Mattsson
a97b0023d4
* feat: import Fortnox voucher attachments * fix: show Fortnox document import follow-up * fix: harden optional Fortnox document import * test: pin optional Fortnox import flow * fix: use browser timer handle type * fix: avoid serializing OAuth resume state
483 lines
16 KiB
TypeScript
483 lines
16 KiB
TypeScript
/**
|
|
* Provider document (underlag) import: best-effort, re-runnable.
|
|
*
|
|
* The migration imports the GL via SIE and the entity registers via the
|
|
* provider API, but the receipts/underlag attached to each verifikat are not
|
|
* carried by either. This step closes that gap for Bokio and Fortnox: it
|
|
* resolves each receipt's target gnubok verifikat from the SIE-preserved
|
|
* provider voucher number, and stores it through the document service
|
|
* (storage + document_attachments), linked to the journal entry.
|
|
*
|
|
* Guarantees:
|
|
* - Idempotent: a receipt already archived for this verifikat (same content
|
|
* AND same journal entry, keyed on company_id + sha256 + journal_entry_id)
|
|
* is skipped, so re-runs don't duplicate. The pair matters: the same file
|
|
* content can legitimately back several verifikat (one arrende contract
|
|
* attached to each year's arrende verifikat), so content alone must not
|
|
* dedup across vouchers. This matters because a receipt linked to a posted
|
|
* verifikat becomes räkenskapsinformation and is undeletable
|
|
* (BFL 7 kap 2§ / WORM triggers).
|
|
* - Best-effort: a per-receipt failure is counted and logged, never thrown,
|
|
* so one bad download can't abort the sweep.
|
|
*
|
|
* Driven from its own /import-documents route rather than the migration's
|
|
* critical path: provider document APIs are rate-limited and a full sweep can
|
|
* issue hundreds of download calls. Fortnox requires archive and connectfile
|
|
* scopes; existing consents must reconnect before this import can run.
|
|
*/
|
|
|
|
import type { SupabaseClient } from '@supabase/supabase-js'
|
|
import { resolveConsent, type ResolvedConsent } from '@/lib/providers/resolve-consent'
|
|
import { BokioClient } from '@/lib/providers/bokio/client'
|
|
import {
|
|
fetchBokioUploads,
|
|
fetchBokioVoucherIndex,
|
|
downloadBokioUpload,
|
|
type BokioUpload,
|
|
} from '@/lib/providers/bokio/attachments'
|
|
import { FortnoxApiError, FortnoxClient } from '@/lib/providers/fortnox/client'
|
|
import {
|
|
downloadFortnoxArchiveFile,
|
|
fetchFortnoxFileConnections,
|
|
fetchFortnoxFinancialYears,
|
|
} from '@/lib/providers/fortnox/attachments'
|
|
import {
|
|
uploadDocument,
|
|
computeSHA256,
|
|
detectFileMagic,
|
|
ALLOWED_DOCUMENT_TYPES,
|
|
} from '@/lib/core/documents/document-service'
|
|
import { fetchAllRows } from '@/lib/supabase/fetch-all'
|
|
import { createLogger } from '@/lib/logger'
|
|
|
|
const log = createLogger('extensions/arcim-migration/import-documents')
|
|
|
|
export class FortnoxDocumentScopesRequiredError extends Error {
|
|
readonly code = 'PROVIDER_DOCUMENT_SCOPES_REQUIRED'
|
|
|
|
constructor() {
|
|
super('Fortnox consent lacks archive/connectfile scope: reconnect required')
|
|
this.name = 'FortnoxDocumentScopesRequiredError'
|
|
}
|
|
}
|
|
|
|
export interface ImportDocumentsOptions {
|
|
supabase: SupabaseClient
|
|
companyId: string
|
|
userId: string
|
|
consentId: string
|
|
/** Resolve + report what would be attached without downloading or writing. */
|
|
dryRun?: boolean
|
|
}
|
|
|
|
export interface ImportDocumentsResult {
|
|
provider: string
|
|
/** Provider attachments linked to a voucher that were considered. */
|
|
scanned: number
|
|
/** Receipts newly archived and linked to their verifikat. */
|
|
linked: number
|
|
/** Receipts already archived for this verifikat (sha256 + journal entry match): re-run skip. */
|
|
skipped: number
|
|
/** Attachments whose provider voucher resolved to no gnubok verifikat. */
|
|
unmatched: number
|
|
/** Receipts that failed to download/validate/store (counted, not thrown). */
|
|
failed: number
|
|
dryRun: boolean
|
|
/** A few unmatched voucher labels, to aid diagnosis without dumping all. */
|
|
unmatchedSamples: { uploadId: string; voucher: string; date: string }[]
|
|
}
|
|
|
|
interface FiscalPeriodRow {
|
|
id: string
|
|
period_start: string
|
|
period_end: string
|
|
}
|
|
|
|
interface VoucherRow {
|
|
id: string
|
|
fiscal_period_id: string
|
|
entry_date: string
|
|
source_voucher_series: string | null
|
|
source_voucher_number: number | null
|
|
}
|
|
|
|
interface ProviderAttachment {
|
|
id: string
|
|
fileName: string | null
|
|
fileNameIsBaseName: boolean
|
|
declaredContentType: string | null
|
|
ref: { series: string; number: number; date: string; dateTo?: string } | null
|
|
}
|
|
|
|
interface ProviderAttachmentSource {
|
|
list(): Promise<ProviderAttachment[]>
|
|
download(id: string): Promise<{ bytes: ArrayBuffer; contentType: string | null }>
|
|
}
|
|
|
|
const EXTENSION_BY_TYPE: Record<string, string> = {
|
|
'application/pdf': 'pdf',
|
|
'image/jpeg': 'jpg',
|
|
'image/png': 'png',
|
|
'image/webp': 'webp',
|
|
}
|
|
|
|
/** Find the fiscal period whose date range contains a given date. */
|
|
function periodIdForDate(periods: FiscalPeriodRow[], date: string): string | null {
|
|
const period = periods.find((p) => p.period_start <= date && date <= p.period_end)
|
|
return period?.id ?? null
|
|
}
|
|
|
|
/**
|
|
* In-memory key for a verifikat: fiscal period + series + number. Scoping by
|
|
* period is essential: providers may reuse voucher numbers across fiscal years.
|
|
*/
|
|
function voucherKey(periodId: string, series: string, number: number): string {
|
|
return `${periodId}|${series}|${number}`
|
|
}
|
|
|
|
/** Remove path/control characters while retaining a readable archive name. */
|
|
function sanitizeProviderFileName(fileName: string): string {
|
|
return (
|
|
fileName
|
|
.trim()
|
|
.replace(/[\u0000-\u001f\u007f<>:"/\\|?*]/g, '_')
|
|
.replace(/_+/g, '_')
|
|
.replace(/^[._ ]+|[. _]+$/g, '')
|
|
.slice(0, 180) || 'file'
|
|
)
|
|
}
|
|
|
|
function normalizedContentType(contentType: string | null): string | null {
|
|
const normalized = contentType?.split(';', 1)[0]?.trim().toLowerCase()
|
|
return normalized || null
|
|
}
|
|
|
|
/** Keep a provider filename when available, otherwise synthesize one. */
|
|
function fileNameFor(
|
|
attachment: ProviderAttachment,
|
|
ref: NonNullable<ProviderAttachment['ref']>,
|
|
contentType: string | null,
|
|
): string {
|
|
const ext = (contentType && EXTENSION_BY_TYPE[contentType]) || 'bin'
|
|
const providerName = attachment.fileName?.trim()
|
|
if (!providerName) return `${ref.series}${ref.number}.${ext}`
|
|
|
|
const sanitized = sanitizeProviderFileName(providerName)
|
|
return !attachment.fileNameIsBaseName && /\.[A-Za-z0-9]+$/.test(sanitized)
|
|
? sanitized
|
|
: `${sanitized}.${ext}`
|
|
}
|
|
|
|
function bokioSource(
|
|
client: BokioClient,
|
|
resolved: ResolvedConsent,
|
|
): ProviderAttachmentSource {
|
|
const { accessToken, providerCompanyId } = resolved
|
|
if (!providerCompanyId) {
|
|
throw new Error('Consent has no provider_company_id: cannot fetch Bokio uploads')
|
|
}
|
|
|
|
return {
|
|
async list() {
|
|
const [uploads, voucherIndex] = await Promise.all([
|
|
fetchBokioUploads(client, accessToken, providerCompanyId),
|
|
fetchBokioVoucherIndex(client, accessToken, providerCompanyId),
|
|
])
|
|
|
|
return uploads
|
|
.filter((upload) => upload.journalEntryId != null)
|
|
.map((upload: BokioUpload): ProviderAttachment => ({
|
|
id: upload.id,
|
|
// Bokio exposes a description rather than a real filename. Treat it
|
|
// as the preferred basename so the adapter preserves current names.
|
|
fileName: upload.description?.trim() || null,
|
|
fileNameIsBaseName: true,
|
|
declaredContentType: upload.contentType,
|
|
ref: voucherIndex.get(upload.journalEntryId as string) ?? null,
|
|
}))
|
|
},
|
|
download(id) {
|
|
return downloadBokioUpload(client, accessToken, providerCompanyId, id)
|
|
},
|
|
}
|
|
}
|
|
|
|
function fortnoxSource(
|
|
client: FortnoxClient,
|
|
accessToken: string,
|
|
): ProviderAttachmentSource {
|
|
return {
|
|
async list() {
|
|
try {
|
|
const financialYears = await fetchFortnoxFinancialYears(client, accessToken)
|
|
const connections = await fetchFortnoxFileConnections(
|
|
client,
|
|
accessToken,
|
|
financialYears.map((year) => year.id),
|
|
)
|
|
const financialYearById = new Map(financialYears.map((year) => [year.id, year]))
|
|
|
|
return connections.map((connection): ProviderAttachment => {
|
|
const financialYear = financialYearById.get(connection.financialYearId)
|
|
return {
|
|
id: connection.fileId,
|
|
fileName: connection.name,
|
|
fileNameIsBaseName: false,
|
|
declaredContentType: null,
|
|
ref: financialYear
|
|
? {
|
|
series: connection.series,
|
|
number: connection.number,
|
|
date: financialYear.fromDate,
|
|
dateTo: financialYear.toDate,
|
|
}
|
|
: null,
|
|
}
|
|
})
|
|
} catch (error) {
|
|
if (error instanceof FortnoxApiError && error.statusCode === 403) {
|
|
throw new FortnoxDocumentScopesRequiredError()
|
|
}
|
|
throw error
|
|
}
|
|
},
|
|
download(id) {
|
|
return downloadFortnoxArchiveFile(client, accessToken, id)
|
|
},
|
|
}
|
|
}
|
|
|
|
export async function importProviderDocuments(
|
|
opts: ImportDocumentsOptions,
|
|
): Promise<ImportDocumentsResult> {
|
|
const { supabase, companyId, userId, consentId, dryRun = false } = opts
|
|
|
|
let resolved = await resolveConsent(companyId, consentId)
|
|
const provider = resolved.consent.provider as string
|
|
|
|
const result: ImportDocumentsResult = {
|
|
provider,
|
|
scanned: 0,
|
|
linked: 0,
|
|
skipped: 0,
|
|
unmatched: 0,
|
|
failed: 0,
|
|
dryRun,
|
|
unmatchedSamples: [],
|
|
}
|
|
|
|
// Unsupported providers are a no-op rather than an error
|
|
// so a mixed-provider caller can invoke this unconditionally.
|
|
if (provider !== 'bokio' && provider !== 'fortnox') {
|
|
log.info('document import skipped: provider not supported', { provider })
|
|
return result
|
|
}
|
|
|
|
const bokioClient = provider === 'bokio' ? new BokioClient() : null
|
|
const fortnoxClient = provider === 'fortnox' ? new FortnoxClient() : null
|
|
const source = (): ProviderAttachmentSource =>
|
|
provider === 'bokio'
|
|
? bokioSource(bokioClient as BokioClient, resolved)
|
|
: fortnoxSource(fortnoxClient as FortnoxClient, resolved.accessToken)
|
|
|
|
// ── Bulk reads (one round of paged requests each, no per-item N+1) ──
|
|
const [attachments, periods, vouchers, existingAttachments] = await Promise.all([
|
|
source().list(),
|
|
// A stable `.order('id')` is required: fetchAllRows pages with `.range()`,
|
|
// and PostgREST paging without a deterministic order can skip or repeat
|
|
// rows once a table exceeds one page (journal_entries crosses 1000 once
|
|
// several years are migrated), which would defeat both resolution and the
|
|
// hash dedup below.
|
|
fetchAllRows<FiscalPeriodRow>(({ from, to }) =>
|
|
supabase
|
|
.from('fiscal_periods')
|
|
.select('id, period_start, period_end')
|
|
.eq('company_id', companyId)
|
|
.order('id', { ascending: true })
|
|
.range(from, to),
|
|
),
|
|
fetchAllRows<VoucherRow>(({ from, to }) =>
|
|
supabase
|
|
.from('journal_entries')
|
|
.select('id, fiscal_period_id, entry_date, source_voucher_series, source_voucher_number')
|
|
.eq('company_id', companyId)
|
|
.not('source_voucher_number', 'is', null)
|
|
.order('id', { ascending: true })
|
|
.range(from, to),
|
|
),
|
|
fetchAllRows<{ sha256_hash: string; journal_entry_id: string | null }>(({ from, to }) =>
|
|
supabase
|
|
.from('document_attachments')
|
|
.select('sha256_hash, journal_entry_id')
|
|
.eq('company_id', companyId)
|
|
.order('id', { ascending: true })
|
|
.range(from, to),
|
|
),
|
|
])
|
|
|
|
// Index gnubok verifikat by (period, series, number) for in-memory resolution.
|
|
const journalEntryByKey = new Map<string, string>()
|
|
const journalEntriesBySourceRef = new Map<string, VoucherRow[]>()
|
|
const ambiguousVoucherKeys = new Set<string>()
|
|
for (const v of vouchers) {
|
|
if (v.source_voucher_series == null || v.source_voucher_number == null) continue
|
|
const sourceRef = `${v.source_voucher_series}|${v.source_voucher_number}`
|
|
journalEntriesBySourceRef.set(sourceRef, [
|
|
...(journalEntriesBySourceRef.get(sourceRef) ?? []),
|
|
v,
|
|
])
|
|
const key = voucherKey(v.fiscal_period_id, v.source_voucher_series, v.source_voucher_number)
|
|
if (journalEntryByKey.has(key)) {
|
|
journalEntryByKey.delete(key)
|
|
ambiguousVoucherKeys.add(key)
|
|
} else if (!ambiguousVoucherKeys.has(key)) {
|
|
journalEntryByKey.set(key, v.id)
|
|
}
|
|
}
|
|
|
|
// (content, verifikat) pairs already archived → idempotent skip set. Keyed
|
|
// on hash + journal entry, NOT hash alone: the same content may back
|
|
// several verifikat and each deserves its own attachment.
|
|
const attachmentKey = (sha256: string, journalEntryId: string) => `${sha256}|${journalEntryId}`
|
|
const seenAttachments = new Set(
|
|
existingAttachments
|
|
.filter((r) => r.journal_entry_id != null)
|
|
.map((r) => attachmentKey(r.sha256_hash, r.journal_entry_id as string)),
|
|
)
|
|
|
|
const recordUnmatched = (uploadId: string, voucher: string, date: string) => {
|
|
result.unmatched++
|
|
if (result.unmatchedSamples.length < 20) {
|
|
result.unmatchedSamples.push({ uploadId, voucher, date })
|
|
}
|
|
}
|
|
|
|
let refreshedAfterUnauthorized = false
|
|
|
|
for (const attachment of attachments) {
|
|
result.scanned++
|
|
const ref = attachment.ref
|
|
|
|
if (!ref) {
|
|
recordUnmatched(attachment.id, '(unresolved)', '')
|
|
continue
|
|
}
|
|
|
|
let journalEntryId: string | undefined
|
|
if (ref.dateTo) {
|
|
const candidates = (journalEntriesBySourceRef.get(`${ref.series}|${ref.number}`) ?? [])
|
|
.filter((voucher) => ref.date <= voucher.entry_date && voucher.entry_date <= ref.dateTo!)
|
|
journalEntryId = candidates.length === 1 ? candidates[0].id : undefined
|
|
} else {
|
|
const periodId = periodIdForDate(periods, ref.date)
|
|
journalEntryId = periodId
|
|
? journalEntryByKey.get(voucherKey(periodId, ref.series, ref.number))
|
|
: undefined
|
|
}
|
|
|
|
if (!journalEntryId) {
|
|
recordUnmatched(attachment.id, `${ref.series}${ref.number}`, ref.date)
|
|
continue
|
|
}
|
|
|
|
if (dryRun) {
|
|
// We can resolve the target without spending a download: count it as a
|
|
// would-link so the preview reflects the real plan.
|
|
result.linked++
|
|
continue
|
|
}
|
|
|
|
const importAttachment = async () => {
|
|
const { bytes, contentType } = await source().download(attachment.id)
|
|
|
|
const sha256 = await computeSHA256(bytes)
|
|
if (seenAttachments.has(attachmentKey(sha256, journalEntryId))) {
|
|
result.skipped++
|
|
return
|
|
}
|
|
|
|
// Trust the bytes over provider metadata: APIs occasionally declare the
|
|
// wrong content type (a JPEG stored as image/png), which
|
|
// would fail magic validation. Sniff the real format first and fall
|
|
// back to the declared type only when no signature is recognised; if
|
|
// neither yields an allowed type, store without a declared type so
|
|
// uploadDocument skips magic validation rather than rejecting.
|
|
const declaredType = normalizedContentType(
|
|
attachment.declaredContentType ?? contentType,
|
|
)
|
|
const sniffedType = detectFileMagic(new Uint8Array(bytes))
|
|
const effectiveType =
|
|
sniffedType ??
|
|
(declaredType && ALLOWED_DOCUMENT_TYPES.includes(declaredType)
|
|
? declaredType
|
|
: undefined)
|
|
|
|
await uploadDocument(
|
|
supabase,
|
|
userId,
|
|
companyId,
|
|
{
|
|
name: fileNameFor(attachment, ref, effectiveType ?? declaredType),
|
|
buffer: bytes,
|
|
type: effectiveType,
|
|
},
|
|
{
|
|
upload_source: 'api',
|
|
journal_entry_id: journalEntryId,
|
|
idempotency_key: journalEntryId,
|
|
},
|
|
)
|
|
|
|
seenAttachments.add(attachmentKey(sha256, journalEntryId))
|
|
result.linked++
|
|
}
|
|
|
|
try {
|
|
await importAttachment()
|
|
} catch (error) {
|
|
let finalError = error
|
|
if (
|
|
provider === 'fortnox' &&
|
|
error instanceof FortnoxApiError &&
|
|
error.statusCode === 401 &&
|
|
!refreshedAfterUnauthorized
|
|
) {
|
|
refreshedAfterUnauthorized = true
|
|
try {
|
|
resolved = await resolveConsent(companyId, consentId)
|
|
await importAttachment()
|
|
continue
|
|
} catch (retryError) {
|
|
finalError = retryError
|
|
}
|
|
}
|
|
|
|
if (
|
|
provider === 'fortnox' &&
|
|
finalError instanceof FortnoxApiError &&
|
|
finalError.statusCode === 403
|
|
) {
|
|
throw new FortnoxDocumentScopesRequiredError()
|
|
}
|
|
|
|
result.failed++
|
|
log.error('failed to import a receipt', finalError as Error, {
|
|
uploadId: attachment.id,
|
|
voucher: `${ref.series}${ref.number}`,
|
|
})
|
|
}
|
|
}
|
|
|
|
log.info('document import complete', {
|
|
companyId,
|
|
dryRun,
|
|
scanned: result.scanned,
|
|
linked: result.linked,
|
|
skipped: result.skipped,
|
|
unmatched: result.unmatched,
|
|
failed: result.failed,
|
|
})
|
|
|
|
return result
|
|
}
|