Add/csv import options (#420)
* feat(import): add customer and supplier parsing functionality - Implemented customer file parsing in `lib/import/customers/parser.ts` with support for Excel and CSV formats. - Created types for detected customer columns and parsed customer rows in `lib/import/customers/types.ts`. - Added tests for customer classification logic in `lib/import/shared/__tests__/classify.test.ts`. - Developed classification functions for customers and suppliers in `lib/import/shared/classify.ts`. - Introduced shared column utility functions in `lib/import/shared/column-utils.ts`. - Implemented supplier file parsing in `lib/import/suppliers/parser.ts` with validation for various fields. - Created types for detected supplier columns and parsed supplier rows in `lib/import/suppliers/types.ts`. - Added tests for supplier column detection and parsing in `lib/import/suppliers/__tests__/column-detector.test.ts` and `lib/import/suppliers/__tests__/parser.test.ts`. * fix(labels): update 'Svenskt företag' to 'Svenskt företag eller organisation' for clarity * feat(import): refactor encoding handling for Swedish files and add tests for character preservation * feat(recapt): implement clearRecaptIdentity function and integrate into logout flow * feat(bookkeeping): implement copy functionality and next voucher sequence retrieval * feat(import): enhance customer and supplier import functionality with normalization and event handling
This commit is contained in:
Mattsson
@@ -0,0 +1,107 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { classifyCustomer, classifySupplier } from '../classify'
|
||||
|
||||
describe('classifyCustomer', () => {
|
||||
it('classifies 12-digit personnummer as individual', () => {
|
||||
expect(classifyCustomer({
|
||||
org_number: '198001011234',
|
||||
vat_number: null,
|
||||
})).toBe('individual')
|
||||
})
|
||||
|
||||
it('classifies 10-digit Swedish org as swedish_business', () => {
|
||||
expect(classifyCustomer({
|
||||
org_number: '5560217780',
|
||||
vat_number: null,
|
||||
})).toBe('swedish_business')
|
||||
})
|
||||
|
||||
it('classifies non-SE EU VAT prefix as eu_business', () => {
|
||||
expect(classifyCustomer({
|
||||
org_number: null,
|
||||
vat_number: 'DE123456789',
|
||||
})).toBe('eu_business')
|
||||
})
|
||||
|
||||
it('classifies non-EU VAT prefix as non_eu_business', () => {
|
||||
expect(classifyCustomer({
|
||||
org_number: null,
|
||||
vat_number: 'NO12345678',
|
||||
})).toBe('non_eu_business')
|
||||
})
|
||||
|
||||
it('keeps SE VAT prefix as swedish_business', () => {
|
||||
expect(classifyCustomer({
|
||||
org_number: '5560217780',
|
||||
vat_number: 'SE556021778001',
|
||||
})).toBe('swedish_business')
|
||||
})
|
||||
|
||||
it('classifies 10-digit personnummer-month-pattern as individual', () => {
|
||||
// Third digit is 0 (month 01), classic personnummer pattern
|
||||
expect(classifyCustomer({
|
||||
org_number: '8001011234',
|
||||
vat_number: null,
|
||||
})).toBe('individual')
|
||||
})
|
||||
|
||||
it('falls back to swedish_business when no signals', () => {
|
||||
expect(classifyCustomer({
|
||||
org_number: null,
|
||||
vat_number: null,
|
||||
})).toBe('swedish_business')
|
||||
})
|
||||
|
||||
it('uses country code when VAT missing', () => {
|
||||
expect(classifyCustomer({
|
||||
org_number: null,
|
||||
vat_number: null,
|
||||
country: 'DE',
|
||||
})).toBe('eu_business')
|
||||
})
|
||||
|
||||
it('detects Norway as non-EU', () => {
|
||||
expect(classifyCustomer({
|
||||
org_number: null,
|
||||
vat_number: null,
|
||||
country: 'Norge',
|
||||
})).toBe('non_eu_business')
|
||||
})
|
||||
})
|
||||
|
||||
describe('classifySupplier', () => {
|
||||
it('never returns individual', () => {
|
||||
expect(classifySupplier({
|
||||
org_number: '198001011234',
|
||||
vat_number: null,
|
||||
})).toBe('swedish_business')
|
||||
})
|
||||
|
||||
it('classifies non-SE EU VAT prefix as eu_business', () => {
|
||||
expect(classifySupplier({
|
||||
org_number: null,
|
||||
vat_number: 'FR12345678901',
|
||||
})).toBe('eu_business')
|
||||
})
|
||||
|
||||
it('classifies post-Brexit GB VAT as non_eu_business', () => {
|
||||
expect(classifySupplier({
|
||||
org_number: null,
|
||||
vat_number: 'GB123456789',
|
||||
})).toBe('non_eu_business')
|
||||
})
|
||||
|
||||
it('classifies XI (Northern Ireland) VAT as eu_business', () => {
|
||||
expect(classifySupplier({
|
||||
org_number: null,
|
||||
vat_number: 'XI123456789',
|
||||
})).toBe('eu_business')
|
||||
})
|
||||
|
||||
it('falls back to swedish_business by default', () => {
|
||||
expect(classifySupplier({
|
||||
org_number: null,
|
||||
vat_number: null,
|
||||
})).toBe('swedish_business')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,79 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import {
|
||||
decodeFileContent,
|
||||
decodeStringContent,
|
||||
hasEncodingIssues,
|
||||
} from '../encoding'
|
||||
|
||||
describe('decodeStringContent', () => {
|
||||
it('recovers UTF-8-as-Latin-1 mojibake for lowercase Swedish chars', () => {
|
||||
expect(decodeStringContent('Malmö')).toBe('Malmö')
|
||||
expect(decodeStringContent('Ã¥re')).toBe('Åre'.toLowerCase())
|
||||
expect(decodeStringContent('Linköping')).toBe('Linköping')
|
||||
})
|
||||
|
||||
it('recovers UTF-8-as-Latin-1 mojibake for uppercase Swedish chars', () => {
|
||||
// The middle char is U+0096 (control), invisible in most renderings → "GÃTEBORG"
|
||||
expect(decodeStringContent('GÃ\u0096TEBORG')).toBe('GÖTEBORG')
|
||||
expect(decodeStringContent('HISINGS KÃ\u0084RRA')).toBe('HISINGS KÄRRA')
|
||||
expect(decodeStringContent('Ã\u0085NGE')).toBe('ÅNGE')
|
||||
})
|
||||
|
||||
it('is a no-op on already-correct Swedish strings', () => {
|
||||
expect(decodeStringContent('GÖTEBORG')).toBe('GÖTEBORG')
|
||||
expect(decodeStringContent('Malmö')).toBe('Malmö')
|
||||
expect(decodeStringContent('STOCKHOLM')).toBe('STOCKHOLM')
|
||||
expect(decodeStringContent('')).toBe('')
|
||||
})
|
||||
|
||||
it('is idempotent (running twice equals running once)', () => {
|
||||
const once = decodeStringContent('Malmö')
|
||||
const twice = decodeStringContent(once)
|
||||
expect(twice).toBe(once)
|
||||
expect(twice).toBe('Malmö')
|
||||
})
|
||||
|
||||
it('preserves non-Swedish strings unchanged', () => {
|
||||
expect(decodeStringContent('Café')).toBe('Café')
|
||||
expect(decodeStringContent('München')).toBe('München')
|
||||
expect(decodeStringContent('123 Main St')).toBe('123 Main St')
|
||||
})
|
||||
})
|
||||
|
||||
describe('hasEncodingIssues', () => {
|
||||
it('detects U+FFFD replacement characters', () => {
|
||||
expect(hasEncodingIssues('Foo\uFFFDbar')).toBe(true)
|
||||
})
|
||||
|
||||
it('detects all six Swedish mojibake patterns', () => {
|
||||
expect(hasEncodingIssues('Malmö')).toBe(true) // ö
|
||||
expect(hasEncodingIssues('Ã¥re')).toBe(true) // å
|
||||
expect(hasEncodingIssues('älg')).toBe(true) // ä
|
||||
expect(hasEncodingIssues('GÃ\u0096TEBORG')).toBe(true) // Ö
|
||||
expect(hasEncodingIssues('Ã\u0085NGE')).toBe(true) // Å
|
||||
expect(hasEncodingIssues('Ã\u0084RRA')).toBe(true) // Ä
|
||||
})
|
||||
|
||||
it('returns false for clean strings', () => {
|
||||
expect(hasEncodingIssues('Stockholm')).toBe(false)
|
||||
expect(hasEncodingIssues('Malmö')).toBe(false)
|
||||
expect(hasEncodingIssues('Café')).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('decodeFileContent', () => {
|
||||
function buf(bytes: number[]): ArrayBuffer {
|
||||
return new Uint8Array(bytes).buffer
|
||||
}
|
||||
|
||||
it('decodes UTF-8 bytes correctly', () => {
|
||||
const utf8 = new TextEncoder().encode('GÖTEBORG').buffer
|
||||
expect(decodeFileContent(utf8)).toBe('GÖTEBORG')
|
||||
})
|
||||
|
||||
it('falls back to Windows-1252 when UTF-8 decode is invalid', () => {
|
||||
// 0xD6 = Ö in Windows-1252; lone 0xD6 is not valid UTF-8 start byte
|
||||
const cp1252 = buf([0x47, 0xd6, 0x54, 0x45, 0x42, 0x4f, 0x52, 0x47])
|
||||
expect(decodeFileContent(cp1252)).toBe('GÖTEBORG')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import * as XLSX from 'xlsx'
|
||||
import { readWorkbookFromBuffer } from '../workbook-reader'
|
||||
|
||||
function bufFromBytes(bytes: number[]): ArrayBuffer {
|
||||
return new Uint8Array(bytes).buffer
|
||||
}
|
||||
|
||||
function rowsOf(workbook: XLSX.WorkBook): string[][] {
|
||||
const sheet = workbook.Sheets[workbook.SheetNames[0]]
|
||||
return XLSX.utils.sheet_to_json(sheet, { header: 1, defval: '', raw: false }) as string[][]
|
||||
}
|
||||
|
||||
describe('readWorkbookFromBuffer', () => {
|
||||
it('decodes UTF-8 CSV with Swedish characters correctly', () => {
|
||||
const csv = new TextEncoder().encode('Namn,Ort\nAcme,GÖTEBORG\nBeta,KÄRRA\n').buffer
|
||||
const wb = readWorkbookFromBuffer(csv, 'lev.csv')
|
||||
expect(rowsOf(wb)).toEqual([
|
||||
['Namn', 'Ort'],
|
||||
['Acme', 'GÖTEBORG'],
|
||||
['Beta', 'KÄRRA'],
|
||||
])
|
||||
})
|
||||
|
||||
it('decodes UTF-8 CSV with BOM', () => {
|
||||
const bom = [0xef, 0xbb, 0xbf]
|
||||
const body = Array.from(new TextEncoder().encode('Namn,Ort\nAcme,GÖTEBORG\n'))
|
||||
const wb = readWorkbookFromBuffer(bufFromBytes([...bom, ...body]), 'lev.csv')
|
||||
expect(rowsOf(wb)).toEqual([
|
||||
['Namn', 'Ort'],
|
||||
['Acme', 'GÖTEBORG'],
|
||||
])
|
||||
})
|
||||
|
||||
it('decodes Windows-1252 CSV with Swedish characters', () => {
|
||||
// "Namn,Ort\nAcme,GÖTEBORG\nBeta,KÄRRA\n" in Windows-1252:
|
||||
// Ö = 0xD6, Ä = 0xC4
|
||||
const bytes = [
|
||||
0x4e, 0x61, 0x6d, 0x6e, 0x2c, 0x4f, 0x72, 0x74, 0x0a,
|
||||
0x41, 0x63, 0x6d, 0x65, 0x2c, 0x47, 0xd6, 0x54, 0x45, 0x42, 0x4f, 0x52, 0x47, 0x0a,
|
||||
0x42, 0x65, 0x74, 0x61, 0x2c, 0x4b, 0xc4, 0x52, 0x52, 0x41, 0x0a,
|
||||
]
|
||||
const wb = readWorkbookFromBuffer(bufFromBytes(bytes), 'lev.csv')
|
||||
expect(rowsOf(wb)).toEqual([
|
||||
['Namn', 'Ort'],
|
||||
['Acme', 'GÖTEBORG'],
|
||||
['Beta', 'KÄRRA'],
|
||||
])
|
||||
})
|
||||
|
||||
it('reads xlsx files via the binary path', () => {
|
||||
const ws = XLSX.utils.aoa_to_sheet([
|
||||
['Namn', 'Ort'],
|
||||
['Acme', 'GÖTEBORG'],
|
||||
])
|
||||
const wb = XLSX.utils.book_new()
|
||||
XLSX.utils.book_append_sheet(wb, ws, 'Sheet1')
|
||||
const buffer = XLSX.write(wb, { type: 'array', bookType: 'xlsx' }) as ArrayBuffer
|
||||
|
||||
const result = readWorkbookFromBuffer(buffer, 'data.xlsx')
|
||||
expect(rowsOf(result)).toEqual([
|
||||
['Namn', 'Ort'],
|
||||
['Acme', 'GÖTEBORG'],
|
||||
])
|
||||
})
|
||||
|
||||
it('treats non-csv extensions as binary spreadsheets', () => {
|
||||
// .xls and .ods both go through the binary path; xlsx handles encoding internally
|
||||
const ws = XLSX.utils.aoa_to_sheet([['A'], ['Ö']])
|
||||
const wb = XLSX.utils.book_new()
|
||||
XLSX.utils.book_append_sheet(wb, ws, 'Sheet1')
|
||||
const buffer = XLSX.write(wb, { type: 'array', bookType: 'xlsx' }) as ArrayBuffer
|
||||
|
||||
const result = readWorkbookFromBuffer(buffer, 'data.xls')
|
||||
expect(rowsOf(result)).toEqual([['A'], ['Ö']])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,106 @@
|
||||
import type { CustomerType, SupplierType } from '@/types'
|
||||
|
||||
/**
|
||||
* EU VAT number prefixes (excluding SE).
|
||||
* Source: https://taxation-ec.europa.eu/online-services/check-vat-number-vies_en
|
||||
*/
|
||||
export const EU_VAT_PREFIXES = new Set([
|
||||
'AT', 'BE', 'BG', 'CY', 'CZ', 'DE', 'DK', 'EE', 'EL', 'ES',
|
||||
'FI', 'FR', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV',
|
||||
'MT', 'NL', 'PL', 'PT', 'RO', 'SI', 'SK', 'XI', // XI = Northern Ireland
|
||||
])
|
||||
|
||||
/**
|
||||
* Strip whitespace, dashes, and dots from an org/personnummer for length checks.
|
||||
*/
|
||||
function digits(value: string | null): string {
|
||||
if (!value) return ''
|
||||
return value.replace(/\D/g, '')
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the third digit of a Swedish org/personnummer.
|
||||
*
|
||||
* Personnummer: month digit (00-12) — third digit ≤ 1.
|
||||
* Företag: third digit ≥ 2 (per Skatteverket's allocation rules).
|
||||
*/
|
||||
function looksLikePersonnummer(orgNumber: string | null): boolean {
|
||||
const d = digits(orgNumber)
|
||||
// 12 digits = full personnummer (YYYYMMDDXXXX)
|
||||
if (d.length === 12) return true
|
||||
// 10 digits — disambiguate by month (positions 3-4 are month, 01-12)
|
||||
if (d.length === 10) {
|
||||
const month = parseInt(d.substring(2, 4), 10)
|
||||
if (month >= 1 && month <= 12 && d[2] <= '1') return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function vatPrefix(vatNumber: string | null): string | null {
|
||||
if (!vatNumber) return null
|
||||
const cleaned = vatNumber.trim().toUpperCase()
|
||||
const match = cleaned.match(/^([A-Z]{2})/)
|
||||
return match ? match[1] : null
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-classify a customer based on org_number + vat_number heuristics.
|
||||
*
|
||||
* Precedence:
|
||||
* 1. Non-SE EU VAT prefix → 'eu_business'
|
||||
* 2. Non-EU letter prefix → 'non_eu_business'
|
||||
* 3. Personnummer-shaped org → 'individual'
|
||||
* 4. Default → 'swedish_business'
|
||||
*/
|
||||
export function classifyCustomer(args: {
|
||||
org_number: string | null
|
||||
vat_number: string | null
|
||||
country?: string | null
|
||||
}): CustomerType {
|
||||
const prefix = vatPrefix(args.vat_number)
|
||||
if (prefix && prefix !== 'SE') {
|
||||
if (EU_VAT_PREFIXES.has(prefix)) return 'eu_business'
|
||||
return 'non_eu_business'
|
||||
}
|
||||
|
||||
const country = args.country?.trim().toUpperCase()
|
||||
if (country && country !== 'SE' && country !== 'SVERIGE' && country !== 'SWEDEN') {
|
||||
// Country-based fallback when VAT is missing.
|
||||
if (country.length === 2 && EU_VAT_PREFIXES.has(country)) return 'eu_business'
|
||||
if (country.length >= 3) {
|
||||
// Common Swedish names for non-EU jurisdictions; heuristic is best-effort
|
||||
if (/norge|norway|usa|kanada|canada|storbritannien|uk|united kingdom/i.test(country)) {
|
||||
return 'non_eu_business'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (looksLikePersonnummer(args.org_number)) return 'individual'
|
||||
return 'swedish_business'
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-classify a supplier. Suppliers cannot be 'individual' — Swedish business
|
||||
* with personnummer is still 'swedish_business' (a sole trader supplier).
|
||||
*/
|
||||
export function classifySupplier(args: {
|
||||
org_number: string | null
|
||||
vat_number: string | null
|
||||
country?: string | null
|
||||
}): SupplierType {
|
||||
const prefix = vatPrefix(args.vat_number)
|
||||
if (prefix && prefix !== 'SE') {
|
||||
if (EU_VAT_PREFIXES.has(prefix)) return 'eu_business'
|
||||
return 'non_eu_business'
|
||||
}
|
||||
|
||||
const country = args.country?.trim().toUpperCase()
|
||||
if (country && country !== 'SE' && country !== 'SVERIGE' && country !== 'SWEDEN') {
|
||||
if (country.length === 2 && EU_VAT_PREFIXES.has(country)) return 'eu_business'
|
||||
if (/norge|norway|usa|kanada|canada|storbritannien|uk|united kingdom/i.test(country)) {
|
||||
return 'non_eu_business'
|
||||
}
|
||||
}
|
||||
|
||||
return 'swedish_business'
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Shared column-detection helpers for register imports
|
||||
* (customers, suppliers, future: articles).
|
||||
*/
|
||||
|
||||
export function normalize(header: string): string {
|
||||
return header.toLowerCase().trim().replace(/[_\-./]/g, ' ')
|
||||
}
|
||||
|
||||
export function matchesKeywords(header: string, keywords: string[]): boolean {
|
||||
const normalized = normalize(header)
|
||||
return keywords.some((kw) => normalized === kw || normalized.includes(kw))
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the first column index whose header matches one of `keywords`,
|
||||
* skipping any indices already taken by other columns.
|
||||
*/
|
||||
export function findColumn(
|
||||
headers: string[],
|
||||
keywords: string[],
|
||||
taken: Set<number>,
|
||||
): number | null {
|
||||
for (let i = 0; i < headers.length; i++) {
|
||||
if (taken.has(i)) continue
|
||||
if (matchesKeywords(headers[i], keywords)) {
|
||||
taken.add(i)
|
||||
return i
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/** Trim a string-or-blank cell, returning null when empty. */
|
||||
export function cellOrNull(value: unknown): string | null {
|
||||
if (value === null || value === undefined) return null
|
||||
const str = String(value).trim()
|
||||
return str === '' ? null : str
|
||||
}
|
||||
|
||||
/** Parse an integer payment term ("30 dagar" → 30) with a default fallback. */
|
||||
export function parsePaymentTerms(value: unknown, fallback: number): number {
|
||||
const str = cellOrNull(value)
|
||||
if (!str) return fallback
|
||||
const match = str.match(/-?\d+/)
|
||||
if (!match) return fallback
|
||||
const n = parseInt(match[0], 10)
|
||||
if (isNaN(n) || n < 0 || n > 365) return fallback
|
||||
return n
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize an org/personal number to its dedup key (digits only).
|
||||
* Returns null for empty input or strings that contain no digits.
|
||||
*/
|
||||
export function normalizeOrgNumber(value: string | null): string | null {
|
||||
if (!value) return null
|
||||
return value.replace(/\D/g, '') || null
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize an email to its dedup key (trimmed + lowercased).
|
||||
* Returns null for empty/whitespace-only input.
|
||||
*/
|
||||
export function normalizeEmail(value: string | null): string | null {
|
||||
if (!value) return null
|
||||
return value.trim().toLowerCase() || null
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
/**
|
||||
* Encoding detection and conversion for Swedish import files.
|
||||
*
|
||||
* Used by bank file, supplier, customer, and opening-balance parsers.
|
||||
* Swedish data exports use either UTF-8 or Windows-1252 (ISO-8859-1).
|
||||
* We detect encoding by checking for valid Swedish characters.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Decode file content, handling both UTF-8 and Windows-1252 encodings.
|
||||
*
|
||||
* Strategy: Try UTF-8 first. If the result contains replacement characters
|
||||
* (U+FFFD) or garbled Swedish chars, fall back to Windows-1252.
|
||||
*/
|
||||
export function decodeFileContent(buffer: ArrayBuffer): string {
|
||||
const utf8Decoder = new TextDecoder('utf-8', { fatal: false })
|
||||
const utf8Result = utf8Decoder.decode(buffer)
|
||||
|
||||
if (!hasEncodingIssues(utf8Result)) {
|
||||
return utf8Result
|
||||
}
|
||||
|
||||
const latin1Decoder = new TextDecoder('windows-1252', { fatal: false })
|
||||
return latin1Decoder.decode(buffer)
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-decode a string that suffered the canonical "UTF-8 bytes read as Latin-1"
|
||||
* mojibake (e.g. "Malmö" → "Malmö", "GÖTEBORG" → "GÖTEBORG").
|
||||
*
|
||||
* Mechanism: each char in the input is a codepoint that was originally a UTF-8
|
||||
* byte misinterpreted as a Latin-1/Windows-1252 character. We pack those chars
|
||||
* back into a byte sequence and decode the bytes as UTF-8 to recover the
|
||||
* original text.
|
||||
*
|
||||
* No-op when the string is already clean (no garbled patterns).
|
||||
*/
|
||||
export function decodeStringContent(content: string): string {
|
||||
if (!hasEncodingIssues(content)) {
|
||||
return content
|
||||
}
|
||||
|
||||
try {
|
||||
const bytes = new Uint8Array(content.length)
|
||||
for (let i = 0; i < content.length; i++) {
|
||||
bytes[i] = content.charCodeAt(i) & 0xff
|
||||
}
|
||||
const decoder = new TextDecoder('utf-8', { fatal: false })
|
||||
return decoder.decode(bytes)
|
||||
} catch {
|
||||
return content
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a string has encoding issues (garbled Swedish characters).
|
||||
*/
|
||||
export function hasEncodingIssues(text: string): boolean {
|
||||
if (text.includes('\uFFFD')) return true
|
||||
|
||||
// Common garbled patterns when Windows-1252 is read as UTF-8:
|
||||
// Ã¥ = å, ä = ä, ö = ö, Ã… = Å, Ä = Ä, Ö = Ö
|
||||
const garbledPatterns = ['Ã¥', 'ä', 'ö', 'Ã\u0085', 'Ã\u0084', 'Ã\u0096']
|
||||
return garbledPatterns.some((pattern) => text.includes(pattern))
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize line endings to \n
|
||||
*/
|
||||
export function normalizeLineEndings(content: string): string {
|
||||
return content.replace(/\r\n/g, '\n').replace(/\r/g, '\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip BOM (Byte Order Mark) from start of content
|
||||
*/
|
||||
export function stripBOM(content: string): string {
|
||||
if (content.charCodeAt(0) === 0xfeff) {
|
||||
return content.slice(1)
|
||||
}
|
||||
return content
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare file content for parsing: strip BOM, normalize line endings, handle encoding
|
||||
*/
|
||||
export function prepareContent(content: string): string {
|
||||
return normalizeLineEndings(stripBOM(decodeStringContent(content)))
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import * as XLSX from 'xlsx'
|
||||
import { decodeFileContent } from './encoding'
|
||||
|
||||
/**
|
||||
* Read a workbook from a raw file buffer, with correct encoding handling
|
||||
* for CSV files.
|
||||
*
|
||||
* For binary spreadsheet formats (.xlsx, .xls, .ods), xlsx handles encoding
|
||||
* via the embedded codepage and we pass the buffer through as `type: 'array'`.
|
||||
*
|
||||
* For CSV files, xlsx with `type: 'array'` decodes bytes as Latin-1, which
|
||||
* mangles UTF-8 multi-byte sequences (e.g. Ö → Ö). We instead detect the
|
||||
* source encoding (UTF-8 with optional BOM, or Windows-1252) and decode to
|
||||
* a string before handing it to xlsx as `type: 'string'`.
|
||||
*/
|
||||
export function readWorkbookFromBuffer(buffer: ArrayBuffer, filename: string): XLSX.WorkBook {
|
||||
const ext = filename.toLowerCase().split('.').pop() ?? ''
|
||||
if (ext === 'csv') {
|
||||
const content = decodeFileContent(buffer)
|
||||
return XLSX.read(content, { type: 'string' })
|
||||
}
|
||||
return XLSX.read(buffer, { type: 'array' })
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the workbook from `buffer` and return raw rows from its largest sheet.
|
||||
*
|
||||
* Picks the sheet with the most rows (a heuristic that handles files where
|
||||
* the header sheet isn't the first one). Returns rows as a 2D string array
|
||||
* with the header row included; cells default to empty string.
|
||||
*/
|
||||
export function readBestSheet(
|
||||
buffer: ArrayBuffer,
|
||||
filename: string,
|
||||
): { sheetName: string; rawData: string[][] } {
|
||||
const workbook = readWorkbookFromBuffer(buffer, filename)
|
||||
|
||||
let bestSheet = workbook.SheetNames[0]
|
||||
let bestRowCount = 0
|
||||
for (const name of workbook.SheetNames) {
|
||||
const sheet = workbook.Sheets[name]
|
||||
const range = XLSX.utils.decode_range(sheet['!ref'] || 'A1')
|
||||
const rowCount = range.e.r - range.s.r + 1
|
||||
if (rowCount > bestRowCount) {
|
||||
bestRowCount = rowCount
|
||||
bestSheet = name
|
||||
}
|
||||
}
|
||||
|
||||
const sheet = workbook.Sheets[bestSheet]
|
||||
const rawData: string[][] = XLSX.utils.sheet_to_json(sheet, {
|
||||
header: 1,
|
||||
defval: '',
|
||||
raw: false,
|
||||
})
|
||||
|
||||
return { sheetName: bestSheet, rawData }
|
||||
}
|
||||
Reference in New Issue
Block a user