From 942ab7636f352611ae8affae77c42126a59ecbf4 Mon Sep 17 00:00:00 2001 From: Mattsson <111893710+mattssonn@users.noreply.github.com> Date: Thu, 16 Apr 2026 16:52:14 +0200 Subject: [PATCH] Migration timeouts (#258) * feat: optimize migration process with bulk reads and chunked inserts to improve performance * feat: add SIE-import requirement banner for specific providers in migration workflow * feat: enhance error handling in TIC API integration with structured HTTP responses * Update components/extensions/general/ArcimMigrationWorkspace.tsx Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --------- Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../general/ArcimMigrationWorkspace.tsx | 24 + .../lib/migration-orchestrator.ts | 594 ++++++++++++------ .../general/tic/__tests__/lookup.test.ts | 47 ++ extensions/general/tic/index.ts | 152 +++-- 4 files changed, 561 insertions(+), 256 deletions(-) diff --git a/components/extensions/general/ArcimMigrationWorkspace.tsx b/components/extensions/general/ArcimMigrationWorkspace.tsx index 8208aba0..6184f9c9 100644 --- a/components/extensions/general/ArcimMigrationWorkspace.tsx +++ b/components/extensions/general/ArcimMigrationWorkspace.tsx @@ -204,9 +204,33 @@ function ProviderStep({ isLoadingStatus: boolean }) { const activeConsents = connectionStatus?.consents.filter(c => c.status === 1) ?? [] + const hasSieImport = (connectionStatus?.sieImports.filter(i => i.status === 'completed').length ?? 0) > 0 + const allFortnox = activeConsents.length > 0 && activeConsents.every(c => c.provider === 'fortnox') + const showSieRequiredBanner = !isLoadingStatus && !hasSieImport && !allFortnox return (
+ {/* SIE-required banner (not relevant for Fortnox — it fetches SIE itself) */} + {showSieRequiredBanner && ( +
+ +
+

SIE-import krävs först

+

+ Bokio, Visma, Björn Lundén och Briox hämtar endast kunder, leverantörer och fakturor via API:et. Bokföringsdata (kontoplan, verifikationer och balanser) måste importeras via SIE-fil först. Gäller inte Fortnox — där hämtar vi SIE direkt via API:et. +

+ + + Ladda upp SIE-fil + + +
+
+ )} + {/* Existing connections */} {activeConsents.length > 0 && ( diff --git a/extensions/general/arcim-migration/lib/migration-orchestrator.ts b/extensions/general/arcim-migration/lib/migration-orchestrator.ts index df3489a7..71aa5a36 100644 --- a/extensions/general/arcim-migration/lib/migration-orchestrator.ts +++ b/extensions/general/arcim-migration/lib/migration-orchestrator.ts @@ -10,11 +10,17 @@ * 3. Suppliers → needed before supplier invoices * 4. Sales invoices (all statuses, duplicates skipped) * 5. Supplier invoices (all statuses, duplicates skipped) + * + * Performance note: All steps use bulk reads + chunked inserts to + * avoid N+1 round-trips that would exhaust the Vercel function + * timeout (300s hard cap). A typical import with a few thousand + * entities completes in a handful of Supabase requests per step. */ import type { SupabaseClient } from '@supabase/supabase-js' import type { MigrationProgress, MigrationResults, SkipReasons } from '../types' import type { ProviderName } from '@/lib/providers/types' +import type { CustomerDto, SupplierDto, SalesInvoiceDto, SupplierInvoiceDto, PartyDto } from '@/lib/providers/dto' import { resolveConsent } from '@/lib/providers/resolve-consent' import { fetchCompanyInfoDirect, @@ -23,6 +29,7 @@ import { fetchSalesInvoicesDirect, fetchSupplierInvoicesDirect, } from '@/lib/providers/provider-data-fetcher' +import { fetchAllRows } from '@/lib/supabase/fetch-all' import { mapCustomer, mapSupplier, @@ -45,10 +52,30 @@ export interface MigrationOptions { onProgress?: (progress: MigrationProgress) => void } +/** + * Chunk size for bulk inserts. 500 rows/request keeps payloads below + * PostgREST's practical size limit while minimising round-trips. + */ +const INSERT_CHUNK_SIZE = 500 + function emitProgress(options: MigrationOptions, progress: MigrationProgress) { options.onProgress?.(progress) } +function chunk(arr: T[], size: number): T[][] { + const out: T[][] = [] + for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size)) + return out +} + +function getOrgNumberFromParty(party: PartyDto): string | null { + return ( + party.legalEntity?.companyId || + party.identifications?.find((i) => i.schemeId === 'SE:ORGNR')?.id || + null + ) +} + // ── Main orchestrator ───────────────────────────────────────────── export async function executeMigration(options: MigrationOptions): Promise { @@ -61,8 +88,6 @@ export async function executeMigration(options: MigrationOptions): Promise() + const orgNumberToCustomerId = new Map() + const nameToCustomerId = new Map() if (options.importCustomers !== false) { emitProgress(options, { status: 'importing', currentStep: 'Importerar kunder...', progress: 20 }) try { const customers = await fetchCustomersDirect(provider, accessToken, providerCompanyId) + + // One bulk read instead of N `.eq('org_number', ...)` lookups. + const existingCustomers = await fetchAllRows<{ id: string; org_number: string | null; name: string | null }>( + ({ from, to }) => + supabase + .from('customers') + .select('id, org_number, name') + .eq('company_id', companyId) + .range(from, to) + ) + for (const row of existingCustomers) { + if (row.org_number) orgNumberToCustomerId.set(row.org_number, row.id) + if (row.name) nameToCustomerId.set(row.name, row.id) + } + let imported = 0 let skipped = 0 const skipReasons: SkipReasons = {} + type PendingCustomer = { + dto: CustomerDto + row: Record + } + const pending: PendingCustomer[] = [] + for (const customer of customers) { if (!customer.active) { - console.log(`[migration] Customer skipped (inactive): ${customer.party.name}`) skipReasons.inactive = (skipReasons.inactive ?? 0) + 1 skipped++ continue } - const orgNumber = customer.party.legalEntity?.companyId || - customer.party.identifications?.find(i => i.schemeId === 'SE:ORGNR')?.id - if (orgNumber) { - const { data: existing } = await supabase - .from('customers') - .select('id') - .eq('company_id', companyId) - .eq('org_number', orgNumber) - .limit(1) - - if (existing && existing.length > 0) { - console.log(`[migration] Customer skipped (duplicate org_number ${orgNumber}): ${customer.party.name}`) - customerIdMap.set(customer.id, existing[0].id) - skipReasons.duplicate = (skipReasons.duplicate ?? 0) + 1 - skipped++ - continue - } + const orgNumber = getOrgNumberFromParty(customer.party) + if (orgNumber && orgNumberToCustomerId.has(orgNumber)) { + customerIdMap.set(customer.id, orgNumberToCustomerId.get(orgNumber)!) + skipReasons.duplicate = (skipReasons.duplicate ?? 0) + 1 + skipped++ + continue } - const mapped = mapCustomer(customer, userId, companyId) + pending.push({ dto: customer, row: mapCustomer(customer, userId, companyId) }) + } + + for (const batch of chunk(pending, INSERT_CHUNK_SIZE)) { + const rows = batch.map((p) => p.row) const { data: inserted, error } = await supabase .from('customers') - .insert(mapped) - .select('id') - .single() + .insert(rows) + .select('id, org_number, name') - if (error || !inserted) { - console.error(`[migration] Customer insert failed: ${customer.party.name}`, error?.message) - skipReasons.failed = (skipReasons.failed ?? 0) + 1 - skipped++ - } else { - customerIdMap.set(customer.id, inserted.id) + if (error) { + console.error(`[migration] Customer batch insert failed (${batch.length} rows):`, error.message) + skipReasons.failed = (skipReasons.failed ?? 0) + batch.length + skipped += batch.length + continue + } + + // PostgREST returns inserted rows in the same order as supplied, + // so we can pair them up by index to recover the provider id. + const insertedRows = inserted ?? [] + for (let i = 0; i < batch.length && i < insertedRows.length; i++) { + const providerId = batch[i].dto.id + const newId = insertedRows[i].id + customerIdMap.set(providerId, newId) + if (insertedRows[i].org_number) orgNumberToCustomerId.set(insertedRows[i].org_number!, newId) + if (insertedRows[i].name) nameToCustomerId.set(insertedRows[i].name!, newId) imported++ } } @@ -165,57 +217,75 @@ export async function executeMigration(options: MigrationOptions): Promise() + const orgNumberToSupplierId = new Map() + const nameToSupplierId = new Map() if (options.importSuppliers !== false) { emitProgress(options, { status: 'importing', currentStep: 'Importerar leverantörer...', progress: 40 }) try { const suppliers = await fetchSuppliersDirect(provider, accessToken, providerCompanyId) + + const existingSuppliers = await fetchAllRows<{ id: string; org_number: string | null; name: string | null }>( + ({ from, to }) => + supabase + .from('suppliers') + .select('id, org_number, name') + .eq('company_id', companyId) + .range(from, to) + ) + for (const row of existingSuppliers) { + if (row.org_number) orgNumberToSupplierId.set(row.org_number, row.id) + if (row.name) nameToSupplierId.set(row.name, row.id) + } + let imported = 0 let skipped = 0 const skipReasons: SkipReasons = {} + type PendingSupplier = { dto: SupplierDto; row: Record } + const pending: PendingSupplier[] = [] + for (const supplier of suppliers) { if (!supplier.active) { - console.log(`[migration] Supplier skipped (inactive): ${supplier.party.name}`) skipReasons.inactive = (skipReasons.inactive ?? 0) + 1 skipped++ continue } - const orgNumber = supplier.party.legalEntity?.companyId || - supplier.party.identifications?.find(i => i.schemeId === 'SE:ORGNR')?.id - if (orgNumber) { - const { data: existing } = await supabase - .from('suppliers') - .select('id') - .eq('company_id', companyId) - .eq('org_number', orgNumber) - .limit(1) - - if (existing && existing.length > 0) { - console.log(`[migration] Supplier skipped (duplicate org_number ${orgNumber}): ${supplier.party.name}`) - supplierIdMap.set(supplier.id, existing[0].id) - skipReasons.duplicate = (skipReasons.duplicate ?? 0) + 1 - skipped++ - continue - } + const orgNumber = getOrgNumberFromParty(supplier.party) + if (orgNumber && orgNumberToSupplierId.has(orgNumber)) { + supplierIdMap.set(supplier.id, orgNumberToSupplierId.get(orgNumber)!) + skipReasons.duplicate = (skipReasons.duplicate ?? 0) + 1 + skipped++ + continue } - const mapped = mapSupplier(supplier, userId, companyId) + pending.push({ dto: supplier, row: mapSupplier(supplier, userId, companyId) }) + } + + for (const batch of chunk(pending, INSERT_CHUNK_SIZE)) { + const rows = batch.map((p) => p.row) const { data: inserted, error } = await supabase .from('suppliers') - .insert(mapped) - .select('id') - .single() + .insert(rows) + .select('id, org_number, name') - if (error || !inserted) { - console.error(`[migration] Supplier insert failed: ${supplier.party.name}`, error?.message) - skipReasons.failed = (skipReasons.failed ?? 0) + 1 - skipped++ - } else { - supplierIdMap.set(supplier.id, inserted.id) + if (error) { + console.error(`[migration] Supplier batch insert failed (${batch.length} rows):`, error.message) + skipReasons.failed = (skipReasons.failed ?? 0) + batch.length + skipped += batch.length + continue + } + + const insertedRows = inserted ?? [] + for (let i = 0; i < batch.length && i < insertedRows.length; i++) { + const providerId = batch[i].dto.id + const newId = insertedRows[i].id + supplierIdMap.set(providerId, newId) + if (insertedRows[i].org_number) orgNumberToSupplierId.set(insertedRows[i].org_number!, newId) + if (insertedRows[i].name) nameToSupplierId.set(insertedRows[i].name!, newId) imported++ } } @@ -226,44 +296,68 @@ export async function executeMigration(options: MigrationOptions): Promise(({ from, to }) => + supabase + .from('invoices') + .select('invoice_number') + .eq('company_id', companyId) + .range(from, to) + ) + const existingInvoiceNumbers = new Set(existingInvoices.map((r) => r.invoice_number)) + let imported = 0 let skipped = 0 const skipReasons: SkipReasons = {} - for (const inv of invoices) { - const customerOrgNumber = inv.customer.legalEntity?.companyId || - inv.customer.identifications?.find(i => i.schemeId === 'SE:ORGNR')?.id + // Phase A: resolve customer for each invoice; collect those that + // need a minimal customer record to be created on-the-fly. + type ResolvedInvoice = { dto: SalesInvoiceDto; customerId: string } + const resolved: ResolvedInvoice[] = [] + type NewCustomerStub = { + key: string // dedupe key (orgNumber or lowercased name) + row: Record + // invoices waiting for this stub's id + waitingInvoiceIndices: number[] + } + const stubByKey = new Map() + const stubsForThisBatch: { orgNumber: string | null; name: string }[] = [] + + for (const inv of invoices) { + if (existingInvoiceNumbers.has(inv.invoiceNumber)) { + skipReasons.duplicate = (skipReasons.duplicate ?? 0) + 1 + skipped++ + continue + } + + const customerOrgNumber = getOrgNumberFromParty(inv.customer) let customerId: string | null = null - if (customerOrgNumber) { - const { data: match } = await supabase - .from('customers') - .select('id') - .eq('company_id', companyId) - .eq('org_number', customerOrgNumber) - .limit(1) - if (match?.[0]) customerId = match[0].id + if (customerOrgNumber && orgNumberToCustomerId.has(customerOrgNumber)) { + customerId = orgNumberToCustomerId.get(customerOrgNumber)! + } else if (nameToCustomerId.has(inv.customer.name)) { + customerId = nameToCustomerId.get(inv.customer.name)! } - if (!customerId) { - const { data: match } = await supabase - .from('customers') - .select('id') - .eq('company_id', companyId) - .eq('name', inv.customer.name) - .limit(1) - if (match?.[0]) customerId = match[0].id + if (customerId) { + resolved.push({ dto: inv, customerId }) + continue } - if (!customerId) { + // Need to create a minimal customer — dedupe by org number first, + // then by name, so invoices sharing a missing party only create + // one stub row. + const key = (customerOrgNumber ?? `name:${inv.customer.name.toLowerCase()}`).trim() + let stub = stubByKey.get(key) + if (!stub) { const customerType = inferTypeFromParty(inv.customer) const minimalCustomer = { user_id: userId, @@ -271,66 +365,104 @@ export async function executeMigration(options: MigrationOptions): Promise 0) { + const stubList = [...stubByKey.values()] + for (const batch of chunk(stubList, INSERT_CHUNK_SIZE)) { + const { data: inserted, error } = await supabase .from('customers') - .insert(minimalCustomer) - .select('id') - .single() - if (created) { - customerId = created.id - } else { - console.error(`[migration] Sales invoice ${inv.invoiceNumber} skipped — could not create customer "${inv.customer.name}":`, custErr?.message) + .insert(batch.map((s) => s.row)) + .select('id, org_number, name') + + if (error) { + console.error( + `[migration] Sales invoice customer stub insert failed (${batch.length} rows):`, + error.message + ) + // Mark invoices waiting on failed stubs as no-match + for (const s of batch) { + for (const idx of s.waitingInvoiceIndices) { + resolved[idx] = { ...resolved[idx], customerId: '__FAILED__' } + } + } + continue + } + + const insertedRows = inserted ?? [] + for (let i = 0; i < batch.length && i < insertedRows.length; i++) { + const newId = insertedRows[i].id + if (insertedRows[i].org_number) orgNumberToCustomerId.set(insertedRows[i].org_number!, newId) + if (insertedRows[i].name) nameToCustomerId.set(insertedRows[i].name!, newId) + for (const idx of batch[i].waitingInvoiceIndices) { + resolved[idx] = { ...resolved[idx], customerId: newId } + } } } + } - if (!customerId) { - console.log(`[migration] Sales invoice ${inv.invoiceNumber} skipped — no customer match for "${inv.customer.name}" (org: ${customerOrgNumber || 'n/a'})`) + // Drop invoices whose customer couldn't be created. + const ready = resolved.filter((r) => { + if (r.customerId === '__FAILED__') { skipReasons.noMatch = (skipReasons.noMatch ?? 0) + 1 skipped++ - continue + return false } + return !!r.customerId + }) - const { data: existingInv } = await supabase + // Phase C: chunk-insert invoices + their line items. + for (const batch of chunk(ready, INSERT_CHUNK_SIZE)) { + const mappedBatch = batch.map((r) => ({ + ...mapSalesInvoice(r.dto, userId, companyId, r.customerId), + dto: r.dto, + })) + + const { data: insertedInvoices, error: invErr } = await supabase .from('invoices') + .insert(mappedBatch.map((m) => m.invoice)) .select('id') - .eq('company_id', companyId) - .eq('invoice_number', inv.invoiceNumber) - .limit(1) - if (existingInv && existingInv.length > 0) { - console.log(`[migration] Sales invoice ${inv.invoiceNumber} skipped — already exists`) - skipReasons.duplicate = (skipReasons.duplicate ?? 0) + 1 - skipped++ + if (invErr) { + console.error(`[migration] Sales invoice batch insert failed (${batch.length}):`, invErr.message) + skipReasons.failed = (skipReasons.failed ?? 0) + batch.length + skipped += batch.length continue } - const { invoice: mappedInvoice, items: mappedItems } = mapSalesInvoice(inv, userId, companyId, customerId) - - const { data: insertedInv, error: invError } = await supabase - .from('invoices') - .insert(mappedInvoice) - .select('id') - .single() - - if (invError || !insertedInv) { - console.error(`[migration] Sales invoice ${inv.invoiceNumber} insert failed:`, invError?.message) - skipReasons.failed = (skipReasons.failed ?? 0) + 1 - skipped++ - continue + const invoiceRows = insertedInvoices ?? [] + const allItems: Record[] = [] + for (let i = 0; i < mappedBatch.length && i < invoiceRows.length; i++) { + const invoiceId = invoiceRows[i].id + for (const item of mappedBatch[i].items) { + allItems.push({ ...item, invoice_id: invoiceId }) + } + imported++ } - if (mappedItems.length > 0) { - const itemsWithInvoiceId = mappedItems.map(item => ({ - ...item, - invoice_id: insertedInv.id, - })) - await supabase.from('invoice_items').insert(itemsWithInvoiceId) + if (allItems.length > 0) { + for (const itemBatch of chunk(allItems, INSERT_CHUNK_SIZE)) { + const { error: itemErr } = await supabase.from('invoice_items').insert(itemBatch) + if (itemErr) { + console.error(`[migration] Sales invoice items insert failed (${itemBatch.length}):`, itemErr.message) + } + } } - - imported++ } results.salesInvoices = { total: invoices.length, imported, skipped, skipReasons } @@ -339,44 +471,81 @@ export async function executeMigration(options: MigrationOptions): Promise(({ from, to }) => + supabase + .from('supplier_invoices') + .select('supplier_invoice_number, supplier_id') + .eq('company_id', companyId) + .range(from, to) + ) + const existingSuppInvKeys = new Set( + existingSuppInv + .filter((r) => r.supplier_invoice_number && r.supplier_id) + .map((r) => `${r.supplier_id}::${r.supplier_invoice_number}`) + ) + + // Compute next arrival number locally. Unique index is + // (company_id, arrival_number); we're the only writer during + // migration so incrementing in-memory is safe. + const { data: maxRow } = await supabase + .from('supplier_invoices') + .select('arrival_number') + .eq('company_id', companyId) + .order('arrival_number', { ascending: false }) + .limit(1) + .maybeSingle() + let nextArrivalNumber = ((maxRow?.arrival_number as number | undefined) ?? 0) + 1 + let imported = 0 let skipped = 0 const skipReasons: SkipReasons = {} - for (const inv of invoices) { - const supplierOrgNumber = inv.supplier.legalEntity?.companyId || - inv.supplier.identifications?.find(i => i.schemeId === 'SE:ORGNR')?.id + type ResolvedSupplierInvoice = { dto: SupplierInvoiceDto; supplierId: string } + const resolved: ResolvedSupplierInvoice[] = [] + type NewSupplierStub = { + key: string + row: Record + waitingInvoiceIndices: number[] + } + const stubByKey = new Map() + + for (const inv of invoices) { + const supplierOrgNumber = getOrgNumberFromParty(inv.supplier) let supplierId: string | null = null - if (supplierOrgNumber) { - const { data: match } = await supabase - .from('suppliers') - .select('id') - .eq('company_id', companyId) - .eq('org_number', supplierOrgNumber) - .limit(1) - if (match?.[0]) supplierId = match[0].id + if (supplierOrgNumber && orgNumberToSupplierId.has(supplierOrgNumber)) { + supplierId = orgNumberToSupplierId.get(supplierOrgNumber)! + } else if (nameToSupplierId.has(inv.supplier.name)) { + supplierId = nameToSupplierId.get(inv.supplier.name)! } - if (!supplierId) { - const { data: match } = await supabase - .from('suppliers') - .select('id') - .eq('company_id', companyId) - .eq('name', inv.supplier.name) - .limit(1) - if (match?.[0]) supplierId = match[0].id + if (supplierId) { + const dupKey = `${supplierId}::${inv.invoiceNumber}` + if (existingSuppInvKeys.has(dupKey)) { + skipReasons.duplicate = (skipReasons.duplicate ?? 0) + 1 + skipped++ + continue + } + resolved.push({ dto: inv, supplierId }) + continue } - if (!supplierId) { + // Need to create a minimal supplier — dedupe the same way as customers. + const key = (supplierOrgNumber ?? `name:${inv.supplier.name.toLowerCase()}`).trim() + let stub = stubByKey.get(key) + if (!stub) { const supplierType = inferTypeFromParty(inv.supplier) const minimalSupplier = { user_id: userId, @@ -385,79 +554,110 @@ export async function executeMigration(options: MigrationOptions): Promise 0) { + const stubList = [...stubByKey.values()] + for (const batch of chunk(stubList, INSERT_CHUNK_SIZE)) { + const { data: inserted, error } = await supabase .from('suppliers') - .insert(minimalSupplier) - .select('id') - .single() - if (created) { - supplierId = created.id - } else { - console.error(`[migration] Supplier invoice ${inv.invoiceNumber} skipped — could not create supplier "${inv.supplier.name}":`, supErr?.message) + .insert(batch.map((s) => s.row)) + .select('id, org_number, name') + + if (error) { + console.error( + `[migration] Supplier invoice supplier stub insert failed (${batch.length}):`, + error.message + ) + for (const s of batch) { + for (const idx of s.waitingInvoiceIndices) { + resolved[idx] = { ...resolved[idx], supplierId: '__FAILED__' } + } + } + continue + } + + const insertedRows = inserted ?? [] + for (let i = 0; i < batch.length && i < insertedRows.length; i++) { + const newId = insertedRows[i].id + if (insertedRows[i].org_number) orgNumberToSupplierId.set(insertedRows[i].org_number!, newId) + if (insertedRows[i].name) nameToSupplierId.set(insertedRows[i].name!, newId) + for (const idx of batch[i].waitingInvoiceIndices) { + resolved[idx] = { ...resolved[idx], supplierId: newId } + } } } + } - if (!supplierId) { - console.log(`[migration] Supplier invoice ${inv.invoiceNumber} skipped — no supplier match for "${inv.supplier.name}" (org: ${supplierOrgNumber || 'n/a'})`) - skipReasons.noMatch = (skipReasons.noMatch ?? 0) + 1 - skipped++ - continue + // After stubs, do a final dedupe pass against existing supplier invoices + // using the now-resolved supplierId. + const ready = resolved.filter((r) => { + if (r.supplierId === '__FAILED__' || !r.supplierId) { + if (r.supplierId === '__FAILED__') { + skipReasons.noMatch = (skipReasons.noMatch ?? 0) + 1 + skipped++ + } + return false } - - const { data: existingInv } = await supabase - .from('supplier_invoices') - .select('id') - .eq('company_id', companyId) - .eq('supplier_invoice_number', inv.invoiceNumber) - .eq('supplier_id', supplierId) - .limit(1) - - if (existingInv && existingInv.length > 0) { - console.log(`[migration] Supplier invoice ${inv.invoiceNumber} skipped — already exists for supplier "${inv.supplier.name}"`) + const dupKey = `${r.supplierId}::${r.dto.invoiceNumber}` + if (existingSuppInvKeys.has(dupKey)) { skipReasons.duplicate = (skipReasons.duplicate ?? 0) + 1 skipped++ - continue + return false } + return true + }) - const { invoice: mappedInvoice, items: mappedItems } = mapSupplierInvoice(inv, userId, companyId, supplierId) + for (const batch of chunk(ready, INSERT_CHUNK_SIZE)) { + const mappedBatch = batch.map((r) => { + const { invoice, items } = mapSupplierInvoice(r.dto, userId, companyId, r.supplierId) + invoice.arrival_number = nextArrivalNumber++ + return { invoice, items, dto: r.dto } + }) - // Get next arrival number (ankomstnummer) — required NOT NULL column - const { data: arrivalNum, error: arrivalError } = await supabase - .rpc('get_next_arrival_number', { p_company_id: companyId }) - - if (arrivalError || arrivalNum == null) { - console.error(`[migration] Supplier invoice ${inv.invoiceNumber} skipped — could not get arrival number:`, arrivalError?.message) - skipReasons.failed = (skipReasons.failed ?? 0) + 1 - skipped++ - continue - } - - mappedInvoice.arrival_number = arrivalNum - - const { data: insertedInv, error: invError } = await supabase + const { data: insertedInvoices, error: invErr } = await supabase .from('supplier_invoices') - .insert(mappedInvoice) + .insert(mappedBatch.map((m) => m.invoice)) .select('id') - .single() - if (invError || !insertedInv) { - console.error(`[migration] Supplier invoice ${inv.invoiceNumber} insert failed for "${inv.supplier.name}":`, invError?.message, JSON.stringify(mappedInvoice, null, 2)) - skipReasons.failed = (skipReasons.failed ?? 0) + 1 - skipped++ + if (invErr) { + console.error(`[migration] Supplier invoice batch insert failed (${batch.length}):`, invErr.message) + skipReasons.failed = (skipReasons.failed ?? 0) + batch.length + skipped += batch.length + // Roll the counter back so we don't leave a huge gap on retry. + nextArrivalNumber -= batch.length continue } - if (mappedItems.length > 0) { - const itemsWithInvoiceId = mappedItems.map(item => ({ - ...item, - supplier_invoice_id: insertedInv.id, - })) - await supabase.from('supplier_invoice_items').insert(itemsWithInvoiceId) + const invoiceRows = insertedInvoices ?? [] + const allItems: Record[] = [] + for (let i = 0; i < mappedBatch.length && i < invoiceRows.length; i++) { + const invoiceId = invoiceRows[i].id + for (const item of mappedBatch[i].items) { + allItems.push({ ...item, supplier_invoice_id: invoiceId }) + } + imported++ } - imported++ + if (allItems.length > 0) { + for (const itemBatch of chunk(allItems, INSERT_CHUNK_SIZE)) { + const { error: itemErr } = await supabase.from('supplier_invoice_items').insert(itemBatch) + if (itemErr) { + console.error(`[migration] Supplier invoice items insert failed (${itemBatch.length}):`, itemErr.message) + } + } + } } results.supplierInvoices = { total: invoices.length, imported, skipped, skipReasons } diff --git a/extensions/general/tic/__tests__/lookup.test.ts b/extensions/general/tic/__tests__/lookup.test.ts index 3f50eec9..3e2282b2 100644 --- a/extensions/general/tic/__tests__/lookup.test.ts +++ b/extensions/general/tic/__tests__/lookup.test.ts @@ -17,6 +17,7 @@ import { getEmails, getPhones, } from '../lib/tic-client' +import { TICAPIError } from '../lib/tic-types' import type { TICCompanyDocument } from '../lib/tic-types' const mockSearch = vi.mocked(searchCompanyByOrgNumber) @@ -142,4 +143,50 @@ describe('TIC lookup route', () => { const { data } = await res.json() expect(data.isCeased).toBe(true) }) + + it('returns 503 when TIC is not configured', async () => { + mockSearch.mockRejectedValue( + new TICAPIError('TIC_API_PROXY_URL is not configured', undefined, 'NOT_CONFIGURED') + ) + const res = await lookupHandler(makeRequest('556036-0793')) + expect(res.status).toBe(503) + }) + + it('returns 429 when TIC rate-limits us', async () => { + mockSearch.mockRejectedValue( + new TICAPIError('Rate limit exceeded', 429, 'RATE_LIMIT_EXCEEDED') + ) + const res = await lookupHandler(makeRequest('556036-0793')) + expect(res.status).toBe(429) + }) + + it('returns 504 when TIC times out', async () => { + mockSearch.mockRejectedValue(new TICAPIError('Request timeout', undefined, 'TIMEOUT')) + const res = await lookupHandler(makeRequest('556036-0793')) + expect(res.status).toBe(504) + }) + + it('returns 400 when upstream rejects the org number (4xx)', async () => { + mockSearch.mockRejectedValue(new TICAPIError('TIC API error: Bad Request', 400)) + const res = await lookupHandler(makeRequest('1234567-1234')) + expect(res.status).toBe(400) + }) + + it('returns 502 when upstream returns 5xx', async () => { + mockSearch.mockRejectedValue(new TICAPIError('TIC API error: Bad Gateway', 502)) + const res = await lookupHandler(makeRequest('556036-0793')) + expect(res.status).toBe(502) + }) + + it('returns 502 when fetch fails (network error)', async () => { + mockSearch.mockRejectedValue(new TICAPIError('Failed to fetch from TIC: ECONNRESET')) + const res = await lookupHandler(makeRequest('556036-0793')) + expect(res.status).toBe(502) + }) + + it('returns 500 for non-TICAPIError unexpected errors', async () => { + mockSearch.mockRejectedValue(new Error('boom')) + const res = await lookupHandler(makeRequest('556036-0793')) + expect(res.status).toBe(500) + }) }) diff --git a/extensions/general/tic/index.ts b/extensions/general/tic/index.ts index 428b7209..60a4b183 100644 --- a/extensions/general/tic/index.ts +++ b/extensions/general/tic/index.ts @@ -43,6 +43,88 @@ function bankAccountTypeLabel(type?: number): string { } } +/** + * Translate any error from the TIC pipeline into a structured HTTP response. + * + * Status mapping: + * - NOT_CONFIGURED → 503 (proxy URL missing) + * - RATE_LIMIT_EXCEEDED → 429 (TIC quota hit) + * - TIMEOUT → 504 (TIC took longer than 15s) + * - upstream 4xx → 400 (TIC rejected the input — typically a malformed org number) + * - upstream 5xx → 502 (TIC outage) + * - other / unknown → 500 + * + * Always logs the cleaned org number so we can correlate failures with input + * in Vercel logs. + */ +function handleTicError( + error: unknown, + log: { error: (msg: string, meta?: unknown) => void } | Console, + route: 'lookup' | 'profile', + orgNumber: string, + fallbackMessage: string +): Response { + if (error instanceof TICAPIError) { + const meta = { + route, + orgNumber, + message: error.message, + statusCode: error.statusCode, + code: error.code, + } + + if (error.code === 'NOT_CONFIGURED') { + log.error(`[tic] ${route}: not configured`, meta) + return NextResponse.json({ error: 'TIC is not configured' }, { status: 503 }) + } + + if (error.code === 'RATE_LIMIT_EXCEEDED') { + log.error(`[tic] ${route}: rate limit exceeded`, meta) + return NextResponse.json({ error: 'Rate limit exceeded, try again later' }, { status: 429 }) + } + + if (error.code === 'TIMEOUT') { + log.error(`[tic] ${route}: upstream timeout`, meta) + return NextResponse.json( + { error: 'TIC service did not respond in time' }, + { status: 504 } + ) + } + + // Upstream returned a non-OK status we surfaced as a TICAPIError + if (typeof error.statusCode === 'number') { + if (error.statusCode >= 400 && error.statusCode < 500) { + log.error(`[tic] ${route}: upstream rejected request`, meta) + return NextResponse.json( + { error: 'Invalid request to TIC (upstream rejected)' }, + { status: 400 } + ) + } + if (error.statusCode >= 500) { + log.error(`[tic] ${route}: upstream error`, meta) + return NextResponse.json( + { error: 'TIC service is temporarily unavailable' }, + { status: 502 } + ) + } + } + + // Network/DNS/parse failure surfaced as a TICAPIError without code or statusCode + log.error(`[tic] ${route}: upstream failure`, meta) + return NextResponse.json( + { error: 'TIC service is temporarily unavailable' }, + { status: 502 } + ) + } + + log.error(`[tic] ${route}: unexpected error`, { + route, + orgNumber, + error: error instanceof Error ? { name: error.name, message: error.message, stack: error.stack } : String(error), + }) + return NextResponse.json({ error: fallbackMessage }, { status: 500 }) +} + export const ticExtension: Extension = { id: 'tic', name: 'Bolagsuppgifter', @@ -65,6 +147,8 @@ export const ticExtension: Extension = { ) } + const cleanedOrgNumber = orgNumber.replace(/[\s-]/g, '') + try { // Phase 1: Search — returns name, address, registration flags const doc = await searchCompanyByOrgNumber(orgNumber) @@ -137,10 +221,10 @@ export const ticExtension: Extension = { // Log Phase 2 failures for debugging if (bankResult.status === 'rejected') { - log.warn('[tic] bank accounts fetch failed', { reason: String(bankResult.reason) }) + log.warn('[tic] bank accounts fetch failed', { orgNumber: cleanedOrgNumber, companyId, reason: String(bankResult.reason) }) } if (sniResult.status === 'rejected') { - log.warn('[tic] SNI codes fetch failed', { reason: String(sniResult.reason) }) + log.warn('[tic] SNI codes fetch failed', { orgNumber: cleanedOrgNumber, companyId, reason: String(sniResult.reason) }) } const result: CompanyLookupResult = { @@ -156,33 +240,7 @@ export const ticExtension: Extension = { return NextResponse.json({ data: result }) } catch (error) { - if (error instanceof TICAPIError) { - log.error('[tic] lookup failed', { - message: error.message, - statusCode: error.statusCode, - code: error.code, - }) - - if (error.code === 'NOT_CONFIGURED') { - return NextResponse.json( - { error: 'TIC is not configured' }, - { status: 503 } - ) - } - - if (error.code === 'RATE_LIMIT_EXCEEDED') { - return NextResponse.json( - { error: 'Rate limit exceeded, try again later' }, - { status: 429 } - ) - } - } - - log.error('[tic] unexpected error', { error: String(error) }) - return NextResponse.json( - { error: 'Failed to look up company' }, - { status: 500 } - ) + return handleTicError(error, log, 'lookup', cleanedOrgNumber, 'Failed to look up company') } }, }, @@ -201,6 +259,8 @@ export const ticExtension: Extension = { ) } + const cleanedOrgNumber = orgNumber.replace(/[\s-]/g, '') + try { const doc = await searchCompanyByOrgNumber(orgNumber) @@ -267,13 +327,13 @@ export const ticExtension: Extension = { // Log Phase 2 failures if (bankResult.status === 'rejected') { - log.warn('[tic] profile: bank accounts fetch failed', { reason: String(bankResult.reason) }) + log.warn('[tic] profile: bank accounts fetch failed', { orgNumber: cleanedOrgNumber, companyId, reason: String(bankResult.reason) }) } if (sniResult.status === 'rejected') { - log.warn('[tic] profile: SNI codes fetch failed', { reason: String(sniResult.reason) }) + log.warn('[tic] profile: SNI codes fetch failed', { orgNumber: cleanedOrgNumber, companyId, reason: String(sniResult.reason) }) } if (reportsResult.status === 'rejected') { - log.warn('[tic] profile: financial reports fetch failed', { reason: String(reportsResult.reason) }) + log.warn('[tic] profile: financial reports fetch failed', { orgNumber: cleanedOrgNumber, companyId, reason: String(reportsResult.reason) }) } const fin = doc.mostRecentFinancialSummary @@ -329,33 +389,7 @@ export const ticExtension: Extension = { return NextResponse.json({ data: profile }) } catch (error) { - if (error instanceof TICAPIError) { - log.error('[tic] profile failed', { - message: error.message, - statusCode: error.statusCode, - code: error.code, - }) - - if (error.code === 'NOT_CONFIGURED') { - return NextResponse.json( - { error: 'TIC is not configured' }, - { status: 503 } - ) - } - - if (error.code === 'RATE_LIMIT_EXCEEDED') { - return NextResponse.json( - { error: 'Rate limit exceeded, try again later' }, - { status: 429 } - ) - } - } - - log.error('[tic] profile unexpected error', { error: String(error) }) - return NextResponse.json( - { error: 'Failed to fetch company profile' }, - { status: 500 } - ) + return handleTicError(error, log, 'profile', cleanedOrgNumber, 'Failed to fetch company profile') } }, },