Fix/import data (#200)

* fix: enhance import data handling and consent management across components

* feat: Enhance SIE import functionality with validation and error handling improvements

- Added validation errors and warnings state management in SIEImportWizard.
- Improved error handling for duplicate, validation, and parsing errors during SIE file import.
- Enhanced user feedback with actionable guidance for common import errors.
- Updated SIEUploadStep to display validation errors and warnings.
- Improved error messages in API routes for better clarity and user experience.
- Added file size and type validation in the SIE parse route.
- Enhanced parsing logic to provide more detailed error messages for unbalanced vouchers and missing amounts.
- Created a new storage bucket for SIE file archival in Supabase with appropriate policies for user access.
- Updated tests to reflect changes in error messages and validation logic.

* fix: Improve type assertion for response in getPage method

* Update extensions/general/arcim-migration/lib/migration-orchestrator.ts

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

* Update supabase/migrations/20260408130000_sie_files_storage_bucket.sql

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

* fix: Add company ID verification for consent handling in accept and disconnect endpoints

---------

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
This commit is contained in:
Mattsson
2026-04-08 18:17:00 +02:00
committed by GitHub
co-authored by greptile-apps[bot]
parent 6c79f21679
commit 211033410c
20 changed files with 529 additions and 126 deletions
+1 -1
View File
@@ -210,7 +210,7 @@ describe('generateImportPreview', () => {
it('passes parse issues to preview', () => {
const parsed = makeParsedFile({
issues: [
{ severity: 'warning', line: 5, message: 'Unknown tag: #FOO', tag: 'FOO' },
{ severity: 'warning', line: 5, message: 'Okänd tagg: #FOO — ignoreras', tag: 'FOO' },
{ severity: 'error', line: 10, message: 'Invalid voucher', tag: 'VER' },
],
})
+10 -10
View File
@@ -328,7 +328,7 @@ describe('parseSIEFile', () => {
const errors = result.issues.filter((i) => i.severity === 'error')
expect(errors.length).toBeGreaterThanOrEqual(1)
expect(errors.some((e) => e.message.includes('not balanced'))).toBe(true)
expect(errors.some((e) => e.message.includes('balanserar inte'))).toBe(true)
})
})
@@ -380,7 +380,7 @@ describe('validateSIEFile', () => {
const validation = validateSIEFile(parsed)
expect(validation.valid).toBe(false)
expect(validation.errors.some((e) => e.includes('not balanced'))).toBe(true)
expect(validation.errors.some((e) => e.includes('balanserar inte'))).toBe(true)
})
it('no longer warns about accounts referenced in #IB since parser auto-adds them', () => {
@@ -424,7 +424,7 @@ describe('validateSIEFile', () => {
const parsed = parseSIEFile(content)
const validation = validateSIEFile(parsed)
expect(validation.warnings.some((w) => w.includes('Opening balances not balanced'))).toBe(true)
expect(validation.warnings.some((w) => w.includes('Ingående balanser balanserar inte'))).toBe(true)
})
it('passes with balanced opening balances', () => {
@@ -432,7 +432,7 @@ describe('validateSIEFile', () => {
const validation = validateSIEFile(parsed)
// IB: 50000 + 100000 + (-150000) = 0 → balanced
const ibWarning = validation.warnings.find((w) => w.includes('Opening balances not balanced'))
const ibWarning = validation.warnings.find((w) => w.includes('Ingående balanser balanserar inte'))
expect(ibWarning).toBeUndefined()
})
})
@@ -592,7 +592,7 @@ describe('parseSIEFile — invalid date handling', () => {
const result = parseSIEFile(content)
// Voucher should not be created because date is invalid
expect(result.vouchers).toHaveLength(0)
expect(result.issues.some((i) => i.severity === 'error' && i.message.includes('Invalid voucher definition'))).toBe(true)
expect(result.issues.some((i) => i.severity === 'error' && i.message.includes('Ogiltig verifikationsdefinition'))).toBe(true)
})
it('accepts valid leap year date Feb 29', () => {
@@ -630,7 +630,7 @@ describe('parseSIEFile — invalid date handling', () => {
const result = parseSIEFile(content)
expect(result.vouchers).toHaveLength(0)
expect(result.issues.some((i) => i.message.includes('Invalid voucher definition'))).toBe(true)
expect(result.issues.some((i) => i.message.includes('Ogiltig verifikationsdefinition'))).toBe(true)
})
})
@@ -649,7 +649,7 @@ describe('parseSIEFile — missing amount handling', () => {
const result = parseSIEFile(content)
expect(result.openingBalances).toHaveLength(0)
expect(result.issues.some((i) => i.severity === 'warning' && i.message.includes('Missing amount in #IB'))).toBe(true)
expect(result.issues.some((i) => i.severity === 'warning' && i.message.includes('Belopp saknas i #IB'))).toBe(true)
})
it('skips #UB with missing amount and adds warning', () => {
@@ -664,7 +664,7 @@ describe('parseSIEFile — missing amount handling', () => {
const result = parseSIEFile(content)
expect(result.closingBalances).toHaveLength(0)
expect(result.issues.some((i) => i.severity === 'warning' && i.message.includes('Missing amount in #UB'))).toBe(true)
expect(result.issues.some((i) => i.severity === 'warning' && i.message.includes('Belopp saknas i #UB'))).toBe(true)
})
it('skips #RES with missing amount and adds warning', () => {
@@ -679,7 +679,7 @@ describe('parseSIEFile — missing amount handling', () => {
const result = parseSIEFile(content)
expect(result.resultBalances).toHaveLength(0)
expect(result.issues.some((i) => i.severity === 'warning' && i.message.includes('Missing amount in #RES'))).toBe(true)
expect(result.issues.some((i) => i.severity === 'warning' && i.message.includes('Belopp saknas i #RES'))).toBe(true)
})
it('skips #TRANS with missing amount and adds warning', () => {
@@ -698,7 +698,7 @@ describe('parseSIEFile — missing amount handling', () => {
const result = parseSIEFile(content)
expect(result.vouchers).toHaveLength(1)
expect(result.vouchers[0].lines).toHaveLength(0)
expect(result.issues.some((i) => i.severity === 'warning' && i.message.includes('Missing amount in #TRANS'))).toBe(true)
expect(result.issues.some((i) => i.severity === 'warning' && i.message.includes('Belopp saknas i #TRANS'))).toBe(true)
})
it('still parses valid #IB lines alongside missing-amount ones', () => {
+1 -1
View File
@@ -1160,7 +1160,7 @@ export async function saveMappings(
await supabase
.from('sie_account_mappings')
.upsert(batch, {
onConflict: 'user_id,source_account',
onConflict: 'company_id,source_account',
})
}
}
+35 -19
View File
@@ -388,7 +388,7 @@ export function parseSIEFile(content: string): ParsedSIEFile {
issues,
'error',
lineNum,
`Voucher ${currentVoucher.series}${currentVoucher.number} is not balanced (diff: ${total.toFixed(2)})`,
`Verifikation ${currentVoucher.series}${currentVoucher.number} balanserar inte (differens: ${total.toFixed(2)} kr)`,
'VER'
)
}
@@ -425,7 +425,7 @@ export function parseSIEFile(content: string): ParsedSIEFile {
case 'SIETYP':
header.sieType = parseInt(fields[1], 10) as SIEType
if (![1, 2, 3, 4].includes(header.sieType)) {
addIssue(issues, 'warning', lineNum, `Unknown SIE type: ${fields[1]}`, tag)
addIssue(issues, 'warning', lineNum, `Okänd SIE-typ: ${fields[1]}. Filen tolkas som SIE4.`, tag)
header.sieType = 4
}
break
@@ -520,7 +520,7 @@ export function parseSIEFile(content: string): ParsedSIEFile {
const amountStr = fields[3]
if (!amountStr || amountStr.trim() === '') {
addIssue(issues, 'warning', lineNum, 'Missing amount in #IB, skipping line', tag)
addIssue(issues, 'warning', lineNum, 'Belopp saknas i #IB — raden hoppas över', tag)
break
}
@@ -540,7 +540,7 @@ export function parseSIEFile(content: string): ParsedSIEFile {
const amountStr = fields[3]
if (!amountStr || amountStr.trim() === '') {
addIssue(issues, 'warning', lineNum, 'Missing amount in #UB, skipping line', tag)
addIssue(issues, 'warning', lineNum, 'Belopp saknas i #UB — raden hoppas över', tag)
break
}
@@ -560,7 +560,7 @@ export function parseSIEFile(content: string): ParsedSIEFile {
const amountStr = fields[3]
if (!amountStr || amountStr.trim() === '') {
addIssue(issues, 'warning', lineNum, 'Missing amount in #RES, skipping line', tag)
addIssue(issues, 'warning', lineNum, 'Belopp saknas i #RES — raden hoppas över', tag)
break
}
@@ -598,7 +598,7 @@ export function parseSIEFile(content: string): ParsedSIEFile {
currentVoucher.signature = parseStringField(fields[6])
}
} else {
addIssue(issues, 'error', lineNum, 'Invalid voucher definition', tag)
addIssue(issues, 'error', lineNum, 'Ogiltig verifikationsdefinition — nummer eller datum kunde inte tolkas', tag)
}
break
}
@@ -615,7 +615,7 @@ export function parseSIEFile(content: string): ParsedSIEFile {
// supplementary history. We skip RTRANS/BTRANS to avoid double-counting
// which would make balanced vouchers appear unbalanced.
if (!currentVoucher) {
addIssue(issues, 'error', lineNum, `${tag} outside of VER block`, tag)
addIssue(issues, 'error', lineNum, `#${tag} utanför verifikationsblock (#VER) — filen kan vara skadad`, tag)
break
}
@@ -635,7 +635,7 @@ export function parseSIEFile(content: string): ParsedSIEFile {
const transAmountStr = fields[fieldIndex]
if (!transAmountStr || transAmountStr.trim() === '') {
addIssue(issues, 'warning', lineNum, `Missing amount in #${tag}, skipping line`, tag)
addIssue(issues, 'warning', lineNum, `Belopp saknas i #${tag} — raden hoppas över`, tag)
break
}
@@ -667,7 +667,7 @@ export function parseSIEFile(content: string): ParsedSIEFile {
default:
// Unknown tag - add info issue for notable ones
if (!['KSUMMA', 'BKOD', 'TAXAR', 'OMFATTN', 'DIM', 'OBJEKT', 'OIB', 'OUB', 'PBUDGET', 'PSALDO'].includes(tag)) {
addIssue(issues, 'info', lineNum, `Unknown tag: #${tag}`, tag)
addIssue(issues, 'info', lineNum, `Okänd tagg: #${tag} — ignoreras`, tag)
}
}
} catch (error) {
@@ -675,7 +675,7 @@ export function parseSIEFile(content: string): ParsedSIEFile {
issues,
'error',
lineNum,
`Error parsing ${tag}: ${error instanceof Error ? error.message : 'Unknown error'}`,
`Fel vid tolkning av #${tag}: ${error instanceof Error ? error.message : 'Okänt fel'}`,
tag
)
}
@@ -734,22 +734,22 @@ export function validateSIEFile(parsed: ParsedSIEFile): ValidationResult {
// Check for SIE type
if (!parsed.header.sieType) {
errors.push('Missing SIE type (#SIETYP)')
errors.push('SIE-typ saknas (#SIETYP). Filen kanske inte är en giltig SIE-fil — kontrollera att du exporterat i rätt format.')
}
// Check for company info
if (!parsed.header.companyName) {
warnings.push('No company name found (#FNAMN)')
warnings.push('Företagsnamn saknas (#FNAMN) — vanligtvis ofarligt men bör kontrolleras')
}
// Check for fiscal year
if (parsed.header.fiscalYears.length === 0) {
errors.push('No fiscal year defined (#RAR)')
errors.push('Inget räkenskapsår definierat (#RAR). Filen saknar information om vilken period bokföringen gäller — kontrollera att exporten inkluderar räkenskapsårsdata.')
}
// Check for accounts
if (parsed.accounts.length === 0) {
warnings.push('No accounts found (#KONTO)')
warnings.push('Inga konton hittades (#KONTO). Om filen bara innehåller saldon (SIE1) är detta normalt.')
}
// Warn if non-BAS kontoplan declared — mapping logic assumes BAS number ranges
@@ -758,20 +758,28 @@ export function validateSIEFile(parsed: ParsedSIEFile): ValidationResult {
const isBAS = planType.startsWith('BAS') || planType === 'EUBAS' || planType === 'EU-BAS'
if (!isBAS) {
warnings.push(
`Kontoplanstyp "${parsed.header.kontoPlanType}" är inte BAS-baserad. Alla kontomappningar bör granskas manuellt.`
`Kontoplanstyp "${parsed.header.kontoPlanType}" är inte BAS-baserad. Automatisk kontomappning kan bli felaktig — granska alla mappningar manuellt i nästa steg.`
)
}
}
// Check for unbalanced vouchers
const unbalancedVouchers: string[] = []
for (const voucher of parsed.vouchers) {
const total = voucher.lines.reduce((sum, l) => sum + l.amount, 0)
if (Math.abs(total) > 0.01) {
errors.push(
`Voucher ${voucher.series}${voucher.number} on ${voucher.date.toISOString().split('T')[0]} is not balanced (diff: ${total.toFixed(2)})`
unbalancedVouchers.push(
`${voucher.series}${voucher.number} (${voucher.date.toISOString().split('T')[0]}, diff: ${total.toFixed(2)} kr)`
)
}
}
if (unbalancedVouchers.length > 0) {
const shown = unbalancedVouchers.slice(0, 5)
const remaining = unbalancedVouchers.length - shown.length
errors.push(
`${unbalancedVouchers.length} verifikation(er) balanserar inte (debet ≠ kredit): ${shown.join(', ')}${remaining > 0 ? ` och ${remaining} till` : ''}. Kontrollera att exporten från källsystemet är komplett.`
)
}
// Check for accounts referenced but not defined
const definedAccounts = new Set(parsed.accounts.map((a) => a.number))
@@ -787,11 +795,19 @@ export function validateSIEFile(parsed: ParsedSIEFile): ValidationResult {
}
}
const undefinedAccounts: string[] = []
for (const account of referencedAccounts) {
if (!definedAccounts.has(account)) {
warnings.push(`Account ${account} referenced but not defined in #KONTO`)
undefinedAccounts.push(account)
}
}
if (undefinedAccounts.length > 0) {
const shown = undefinedAccounts.slice(0, 10)
const remaining = undefinedAccounts.length - shown.length
warnings.push(
`${undefinedAccounts.length} konto(n) används i verifikationer men definieras inte i #KONTO: ${shown.join(', ')}${remaining > 0 ? ` och ${remaining} till` : ''}. Kontona skapas automatiskt vid import.`
)
}
// Check opening balance is balanced (for balance sheet accounts)
const ibTotal = parsed.openingBalances
@@ -799,7 +815,7 @@ export function validateSIEFile(parsed: ParsedSIEFile): ValidationResult {
.reduce((sum, b) => sum + b.amount, 0)
if (Math.abs(ibTotal) > 0.01) {
warnings.push(`Opening balances not balanced (diff: ${ibTotal.toFixed(2)})`)
warnings.push(`Ingående balanser balanserar inte (differens: ${ibTotal.toFixed(2)} kr). En automatisk justeringspost mot konto 2099 skapas vid import.`)
}
// Add parse issues as errors/warnings
+38 -4
View File
@@ -1,6 +1,9 @@
import { TokenBucketRateLimiter } from '../rate-limiter';
import { withRetry } from '../retry';
import { BOKIO_BASE_URL, BOKIO_RATE_LIMIT } from './config';
import { createLogger } from '@/lib/logger';
const log = createLogger('bokio-client');
export class BokioApiError extends Error {
constructor(
@@ -24,7 +27,8 @@ function isRetryableError(error: unknown): boolean {
}
interface BokioPaginatedResponse<T> {
items: T[];
items?: T[];
data?: T[]; // Some Bokio endpoints use 'data' instead of 'items'
totalItems: number;
totalPages: number;
currentPage: number;
@@ -72,7 +76,8 @@ export class BokioClient {
/**
* Fetch a paginated list endpoint.
* Bokio returns `{ data: [...], pagination: { page, pageSize, totalPages, totalCount } }`.
* Bokio returns `{ items: [...], totalItems, totalPages, currentPage }`.
* Some endpoints may use `data` instead of `items`.
*/
async getPage<T>(
accessToken: string,
@@ -94,12 +99,41 @@ export class BokioClient {
const path = `/companies/${companyId}${relativePath}?${params.toString()}`;
const response = await this.get<BokioPaginatedResponse<T>>(accessToken, path);
return {
items: Array.isArray(response.items) ? response.items : [],
// Bokio uses 'items' for most endpoints but 'data' for some (e.g., credit notes)
const items = Array.isArray(response.items)
? response.items
: Array.isArray(response.data)
? response.data
: [];
const result = {
items,
page: response.currentPage ?? (options?.page ?? 1),
totalPages: response.totalPages ?? 1,
totalCount: response.totalItems ?? 0,
};
log.info(
`getPage ${relativePath} page=${result.page}/${result.totalPages}: ` +
`${result.items.length} items (totalCount=${result.totalCount})` +
(result.items.length === 0 && result.totalCount > 0
? ` — WARNING: 0 items despite totalCount=${result.totalCount}, raw keys: ${Object.keys(response).join(', ')}`
: ''),
);
// Extra diagnostic: if no items found and response has unexpected keys, log them
if (result.items.length === 0) {
const rawObj = response as unknown as Record<string, unknown>;
const keys = Object.keys(rawObj).filter(k => !['totalItems', 'totalPages', 'currentPage', 'items', 'data'].includes(k));
if (keys.length > 0) {
log.warn(
`Unexpected response keys for ${relativePath}: ${keys.join(', ')}. ` +
`Values: ${keys.map(k => `${k}=${typeof rawObj[k] === 'object' ? JSON.stringify(rawObj[k]).slice(0, 200) : rawObj[k]}`).join(', ')}`,
);
}
}
return result;
}
/**
+7 -3
View File
@@ -133,11 +133,13 @@ export function mapBokioToCustomer(raw: Record<string, unknown>): CustomerDto {
return {
id: String(raw['id'] ?? ''),
customerNumber: String(raw['id'] ?? ''),
type: raw['type'] === 'individual' ? 'private' : 'company',
type: (raw['type'] === 'individual' || raw['type'] === 'person') ? 'private' : 'company',
party,
active: true,
vatNumber: raw['vatNumber'] as string | undefined,
defaultPaymentTermsDays: raw['paymentTerms'] != null ? Number(raw['paymentTerms']) : undefined,
defaultPaymentTermsDays: raw['paymentTerms'] != null && !isNaN(Number(raw['paymentTerms']))
? Number(raw['paymentTerms'])
: undefined,
_raw: raw,
};
}
@@ -231,7 +233,9 @@ export function mapBokioToSupplier(raw: Record<string, unknown>): SupplierDto {
bankAccount: raw['bankAccount'] as string | undefined,
bankGiro: raw['bankgiro'] as string | undefined,
plusGiro: raw['plusgiro'] as string | undefined,
defaultPaymentTermsDays: raw['paymentTerms'] != null ? Number(raw['paymentTerms']) : undefined,
defaultPaymentTermsDays: raw['paymentTerms'] != null && !isNaN(Number(raw['paymentTerms']))
? Number(raw['paymentTerms'])
: undefined,
_raw: raw,
};
}
+36 -7
View File
@@ -13,7 +13,7 @@ import { VismaClient } from './visma/client';
import { VISMA_RESOURCE_CONFIGS } from './visma/config';
import { BrioxClient } from './briox/client';
import { BRIOX_RESOURCE_CONFIGS } from './briox/config';
import { BokioClient } from './bokio/client';
import { BokioClient, BokioApiError } from './bokio/client';
import { BOKIO_RESOURCE_CONFIGS } from './bokio/config';
import { BjornLundenClient } from './bjornlunden/client';
import { BL_RESOURCE_CONFIGS } from './bjornlunden/config';
@@ -44,6 +44,7 @@ async function bokioPaginate<T>(
page++;
} while (page <= totalPages);
console.log(`[bokio-paginate] ${path}: fetched ${allItems.length} total items across ${totalPages} page(s)`);
return allItems;
}
@@ -143,8 +144,14 @@ export async function fetchCustomersDirect(
if (provider === 'bokio') {
const config = BOKIO_RESOURCE_CONFIGS[ResourceType.Customers];
if (!config || !providerCompanyId) return [];
if (!config || !providerCompanyId) {
console.warn(`[provider-data-fetcher] Bokio customers: skipped — config=${!!config}, providerCompanyId=${providerCompanyId ?? 'undefined'}`);
return [];
}
const items = await bokioPaginate<Record<string, unknown>>(accessToken, providerCompanyId, config.listEndpoint);
if (items.length > 0) {
console.log(`[provider-data-fetcher] Bokio customers: first item keys: ${Object.keys(items[0]).join(', ')}`);
}
return items.map((item) => config.mapper(item) as CustomerDto);
}
@@ -186,8 +193,16 @@ export async function fetchSuppliersDirect(
if (provider === 'bokio') {
const config = BOKIO_RESOURCE_CONFIGS[ResourceType.Suppliers];
if (!config || !providerCompanyId) return [];
const items = await bokioPaginate<Record<string, unknown>>(accessToken, providerCompanyId, config.listEndpoint);
return items.map((item) => config.mapper(item) as SupplierDto);
try {
const items = await bokioPaginate<Record<string, unknown>>(accessToken, providerCompanyId, config.listEndpoint);
return items.map((item) => config.mapper(item) as SupplierDto);
} catch (err) {
if (err instanceof BokioApiError && err.statusCode === 404) {
console.log('[provider-data-fetcher] Bokio suppliers endpoint not available (404) — skipping');
return [];
}
throw err;
}
}
if (provider === 'bjornlunden') {
@@ -227,8 +242,14 @@ export async function fetchSalesInvoicesDirect(
if (provider === 'bokio') {
const config = BOKIO_RESOURCE_CONFIGS[ResourceType.SalesInvoices];
if (!config || !providerCompanyId) return [];
if (!config || !providerCompanyId) {
console.warn(`[provider-data-fetcher] Bokio invoices: skipped — config=${!!config}, providerCompanyId=${providerCompanyId ?? 'undefined'}`);
return [];
}
const items = await bokioPaginate<Record<string, unknown>>(accessToken, providerCompanyId, config.listEndpoint);
if (items.length > 0) {
console.log(`[provider-data-fetcher] Bokio invoices: first item keys: ${Object.keys(items[0]).join(', ')}`);
}
return items.map((item) => config.mapper(item) as SalesInvoiceDto);
}
@@ -270,8 +291,16 @@ export async function fetchSupplierInvoicesDirect(
if (provider === 'bokio') {
const config = BOKIO_RESOURCE_CONFIGS[ResourceType.SupplierInvoices];
if (!config || !providerCompanyId) return [];
const items = await bokioPaginate<Record<string, unknown>>(accessToken, providerCompanyId, config.listEndpoint);
return items.map((item) => config.mapper(item) as SupplierInvoiceDto);
try {
const items = await bokioPaginate<Record<string, unknown>>(accessToken, providerCompanyId, config.listEndpoint);
return items.map((item) => config.mapper(item) as SupplierInvoiceDto);
} catch (err) {
if (err instanceof BokioApiError && err.statusCode === 404) {
console.log('[provider-data-fetcher] Bokio supplier-invoices endpoint not available (404) — skipping');
return [];
}
throw err;
}
}
if (provider === 'bjornlunden') {
+3 -2
View File
@@ -28,8 +28,9 @@ export async function resolveConsent(companyId: string, consentId: string): Prom
}
const consent = consentRows[0]!;
if (consent.status !== 1) {
throw { status: 403, message: 'Consent is not in Accepted status' };
// Accept status 0 (token submitted, migration pending) and 1 (fully accepted)
if (consent.status !== 0 && consent.status !== 1) {
throw { status: 403, message: 'Consent is not in a valid status' };
}
if (!consent.provider) {