From 8e1f9d5201df69bc19dbff809374ff4d33cee0b4 Mon Sep 17 00:00:00 2001 From: Jakob Wennberg Date: Sat, 5 Sep 2026 10:20:10 +0200 Subject: [PATCH] fix(migration): complete the rows of migrated sales invoices the hydration budget did not reach (#2291) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(migration): complete the rows of migrated sales invoices the hydration budget did not reach The migration maps sales invoices from the provider's list payload and hydrates the detail form (rows, net, VAT) inside a fixed 90 s budget, open invoices first. Fortnox, Briox and Björn Lundén ship no rows in a list response, so every invoice the budget did not reach was imported as a header with a total and no invoice_items, and nothing ever came back for it: the wizard never showed the hydration report, so the user found out on the invoice page. Measured on prod today: Profilio 384 of 384 (migrated before hydration existed), Loftux 311 of 672, Damac 182 of 542, Clearstoq 1 125 of 1 125. - lib/providers: hydrateSalesInvoices() hydrates a caller-chosen subset of an already-listed register, so a follow-up can spend its budget on the invoices still incomplete on our side instead of re-walking the register open-first and never reaching the rest. - arcim-migration: completeMigratedInvoiceLines() starts from OUR row-less non-draft invoices, joins them to the provider register on number + date (unique on both sides), hydrates only that subset and writes each invoice's rows once the detail total matches the stored total to the öre. The header VAT split is rewritten only when the stored one holds no evidence (null rate, or a non-zero rate label beside 0 kr VAT and subtotal = total). Never the total, status, payments or a journal entry. - Hourly cron (/api/extensions/arcim-migration/complete-invoice-lines/cron, vercel.json + Docker crontabs) drives the pass over consents accepted in the last 60 days, newest first, with a per-company share of the run. - The wizard's result screen now shows "x av y fakturor hämtade med rader" and that the rest are fetched in the background within the hour. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DG5aYcshzKJ1EA7PPhGtVf * fix(migration): write the header VAT fill as a literal, raise the schema-guard ceiling for the row inserts The phantom-column scanner resolves only object-literal payloads. The header update is now a literal (so its six columns are checked); the two invoice_items inserts are runtime row arrays from mapSalesInvoiceLine, the same shape the orchestrator already inserts, so the ceiling moves 399 to 401 with the reason recorded beside the earlier ones. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DG5aYcshzKJ1EA7PPhGtVf * fix(migration): gate the completion cron on token freshness, not consent age, and visit every usable consent Two review findings held. Prod holds 57 accepted consents from the last 60 days, so a fixed page of the newest 25 would leave older companies with row-less invoices waiting behind companies that are already done: the cap is gone (a company with nothing left costs one query and no provider call). And the consent's created_at said nothing about whether its credentials still work: Fortnox refresh tokens live 45 days and rotate on every refresh, so eligibility is now read off the token row (access token expired within the last 45 days, or no expiry at all), which also stops a dead consent from being retried every hour. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DG5aYcshzKJ1EA7PPhGtVf --------- Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 --- DECISIONS.md | 1 + .../cron/__tests__/route.test.ts | 219 ++++++++++ .../complete-invoice-lines/cron/route.ts | 172 ++++++++ .../general/ArcimMigrationWorkspace.tsx | 15 + docker/crontab.hosted | 1 + docker/crontab.self-hosted | 1 + .../__tests__/complete-invoice-lines.test.ts | 370 ++++++++++++++++ .../lib/complete-invoice-lines.ts | 409 ++++++++++++++++++ .../__tests__/hydrate-invoices.test.ts | 62 ++- lib/providers/provider-data-fetcher.ts | 24 + messages/en.json | 3 + messages/sv.json | 3 + tests/schema/no-phantom-columns.test.ts | 9 +- vercel.json | 4 + 14 files changed, 1291 insertions(+), 2 deletions(-) create mode 100644 app/api/extensions/arcim-migration/complete-invoice-lines/cron/__tests__/route.test.ts create mode 100644 app/api/extensions/arcim-migration/complete-invoice-lines/cron/route.ts create mode 100644 extensions/general/arcim-migration/lib/__tests__/complete-invoice-lines.test.ts create mode 100644 extensions/general/arcim-migration/lib/complete-invoice-lines.ts diff --git a/DECISIONS.md b/DECISIONS.md index 410dd590..a0ceb379 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -1586,4 +1586,5 @@ One line per decision: `[YYYY-MM-DD] : `. Appended by agents and [2026-09-04] company_settings.pays_salaries (and a never-attested employer_registered) now follow the ledger: trigger salary_runs_booked_marks_employer (20260904191000) sets them when a salary run is booked, with a backfill for the 12 companies already booking payroll with the flag at its default. Support case 2026-09-04: the assistant told a payroll-running aktiebolag in every answer that it "betalar inte löner" because lib/agent/ask/snapshot.ts asserted the column default (NOT NULL DEFAULT false, only written by the Skatt settings form) as a fact. The snapshot and the composer's KÄNDA FAKTA now apply the employee-facts doctrine (positive evidence or an attested negative, never the default) and the snapshot names Inställningar > Skatt / > Bokföring so the model can point at the page. Chose a trigger over calling a helper from the booking code because there are three booking writers (lib/salary/book-run.ts, the v1 route's own copy, seeders) and 20260717151000 already treated in-app payroll as employer evidence once as a one-off backfill; the trigger makes that rule continuous. An explicit employer_registered = false stays the user's answer. [2026-09-04] Supplier credit notes rest at 'credited' from creation (one row builder in lib/supplier-invoices/credit-note.ts for the dashboard route, the MCP executor and the v1 API) and a CHECK (supplier_invoices_credit_note_not_payable, 20260904190000) keeps every writer out of the payable states; the worklist attest count also excludes is_credit_note. Support case 2026-09-04: a credit note was inserted at 'registered' (the attest entry state) while the detail page rightly had no attest button for it, so Att göra showed "1 leverantörsfaktura att attestera" that nobody could clear; 14 such rows on prod plus one MCP-approved credit note, all backfilled by the migration (one immutable reset-source row skipped, hence NOT VALID). Chose the DB CHECK over fixing only the three insert literals because the invariant is "a credit note is never a payable", not "this literal says registered"; chose 'credited' over 'approved'+approved_at (would surface Markera betald and the Att betala tab) and over 'paid' (the customer side forbids paid credit notes by CHECK); the provider importers already used 'credited'. Also: the GET route now hydrates credited_original with a second scoped query because PostgREST cannot pick a direction for a self-referencing embed hint and returned the one-to-many side (an empty array), rendered as "Krediterar: Ankomst #" with no number. [2026-09-04] Underlag attach on a folder-picked Fortnox export (Loftux, 50 of 50 files refused with UNDERLAG_REF_MISMATCH): the multipart filename is reduced to its basename at the route boundary (lib/documents/upload-file-name.ts), rather than teaching the voucher-ref parser to strip directories or adding a client-supplied file_name field. Chrome writes webkitRelativePath as the multipart filename for folder selections, so the attach check saw "2026/06/Leverantörsfakturor/A166_x.pdf" while the preview had resolved File.name "A166_x.pdf"; the two endpoints received the same file under two names and the guard compared them. Stripping inside the parser would turn a typed manual ref "2024/01/31" into voucher 31 (the manual box shares the parser), and a second client-supplied name is no more trustworthy than the first, so the boundary is the only level that fixes the class. +[2026-09-04] Migrated sales invoices without rows (Profilio 384/384, Loftux 311/672, Damac 182/542, Clearstoq 1 125/1 125): completed by an hourly re-runnable pass (extensions/general/arcim-migration/lib/complete-invoice-lines.ts, cron /api/extensions/arcim-migration/complete-invoice-lines/cron) that starts from OUR row-less invoices, joins them to the provider register on number + date, hydrates only that subset and writes rows once the detail total matches the stored total to the öre; the header VAT split is rewritten only when the stored one holds no evidence (null rate, or 0 kr VAT beside subtotal = total). Why not a bigger in-run budget: the largest register (1 911 invoices at Fortnox's platform-wide 4 req/s) does not fit one 300 s function whatever the split, and a budget-bounded one-shot pass leaves whatever it misses missing forever, silently (the wizard never showed the hydration report; it does now). Why not re-running fetchSalesInvoicesHydrated: it sorts the whole register open-first every time, so a second run re-spends its budget on the same invoices and never reaches the rest. Why not reset + re-import or an arithmetic backfill: reset deletes rows that payments and vouchers already point at, and total/1,25 asserts a rate the source never stated (DECISIONS 2026-08-22). The pass reuses mapSalesInvoice, so a row it writes is indistinguishable from a fully hydrated import; it never touches totals, status, payments or any journal entry (momsdeklaration and every report read the ledger). [2026-09-04] Connector-hop failures (timeout, error envelope, wire-contract mismatch) are transient in every sync path: the row keeps its status and the user message says no renewal is needed, same as AspspUnavailableError (#2202), and the cron now treats AspspUnavailableError the same way instead of parking it in 'error'. Why: on 2026-09-04 the Connect service answered a shape the client rejects and the cron flipped four canary companies to 'error' with SYNC_FAILED_MESSAGE, so users re-authorized consents that were fine. The Zod issues are logged (field paths) because a bare 'unexpected shape' left the failure undiagnosable. Rejected: a new 'degraded' connection status (one more state every filter and the probe would have to learn; the health probe already catches a dead session on the same run) and removing the canary companies from the env (hides the contract bug instead of exposing its field paths). diff --git a/app/api/extensions/arcim-migration/complete-invoice-lines/cron/__tests__/route.test.ts b/app/api/extensions/arcim-migration/complete-invoice-lines/cron/__tests__/route.test.ts new file mode 100644 index 00000000..94d2a755 --- /dev/null +++ b/app/api/extensions/arcim-migration/complete-invoice-lines/cron/__tests__/route.test.ts @@ -0,0 +1,219 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { NextResponse } from 'next/server' + +/** + * The hourly pass that fills in the rows the migration's bounded hydration + * did not reach. The route is thin: refuse without the cron secret, refuse + * when the extension is off, then hand each recent accepted consent to the + * pass with its share of the run and add the counts up. + */ + +vi.mock('@/lib/extensions/loader', () => ({ loadExtensions: vi.fn() })) +vi.mock('@/lib/extensions/registry', () => ({ extensionRegistry: { get: vi.fn() } })) +vi.mock('@/lib/auth/cron', () => ({ verifyCronSecret: vi.fn().mockReturnValue(null) })) + +const h = vi.hoisted(() => ({ + consents: { data: [] as unknown[] | null, error: null as { message: string } | null }, +})) + +vi.mock('@/lib/auth/api-keys', () => ({ + createServiceClientNoCookies: vi.fn(() => ({ + from: vi.fn(() => { + const builder: Record = {} + for (const method of ['select', 'eq', 'not', 'order']) { + builder[method] = vi.fn(() => builder) + } + builder.limit = vi.fn(() => Promise.resolve(h.consents)) + return builder + }), + })), +})) + +vi.mock('@/extensions/general/arcim-migration/lib/complete-invoice-lines', () => ({ + completeMigratedInvoiceLines: vi.fn(), +})) + +import { GET, maxDuration, consentIsUsable } from '../route' +import { extensionRegistry } from '@/lib/extensions/registry' +import { verifyCronSecret } from '@/lib/auth/cron' +import { completeMigratedInvoiceLines } from '@/extensions/general/arcim-migration/lib/complete-invoice-lines' + +const mockRegistryGet = vi.mocked(extensionRegistry.get) +const mockVerifyCronSecret = vi.mocked(verifyCronSecret) +const mockComplete = vi.mocked(completeMigratedInvoiceLines) + +const EMPTY = { + candidates: 0, providerInvoices: 0, matched: 0, unmatched: 0, completed: 0, headersUpdated: 0, + totalMismatch: 0, noLinesAtProvider: 0, notHydrated: 0, vatUnresolved: 0, failed: 0, remaining: 0, + hydration: { needed: 0, hydrated: 0, failed: 0, skippedForBudget: 0 }, dryRun: false, +} + +const DAY_MS = 24 * 60 * 60 * 1000 + +/** A consent whose access token expired `daysAgo` days ago (null: never expires). */ +function consent(id: string, companyId: string, daysAgo: number | null, provider = 'fortnox') { + return { + id, + company_id: companyId, + provider, + created_at: '2026-08-13T22:58:24Z', + provider_consent_tokens: { + token_expires_at: daysAgo === null ? null : new Date(Date.now() - daysAgo * DAY_MS).toISOString(), + }, + } +} + +function makeRequest() { + return new Request('http://localhost/api/extensions/arcim-migration/complete-invoice-lines/cron', { + headers: { authorization: 'Bearer synthetic-cron-secret' }, + }) +} + +beforeEach(() => { + vi.clearAllMocks() + mockVerifyCronSecret.mockReturnValue(null) + mockRegistryGet.mockReturnValue({ id: 'arcim-migration' } as never) + h.consents = { data: [], error: null } +}) + +describe('GET /api/extensions/arcim-migration/complete-invoice-lines/cron', () => { + it('reserves the full function window: hydration is rate-limited at the provider', () => { + expect(maxDuration).toBe(300) + }) + + it('returns 401 when the cron secret is rejected', async () => { + mockVerifyCronSecret.mockReturnValue(NextResponse.json({ error: 'Unauthorized' }, { status: 401 })) + + const response = await GET(makeRequest()) + + expect(response.status).toBe(401) + expect(mockComplete).not.toHaveBeenCalled() + }) + + it('returns 503 EXTENSION_DISABLED when the extension is not in the registry', async () => { + mockRegistryGet.mockReturnValue(undefined) + + const response = await GET(makeRequest()) + const body = await response.json() + + expect(response.status).toBe(503) + expect(body.code).toBe('EXTENSION_DISABLED') + expect(mockComplete).not.toHaveBeenCalled() + }) + + it('runs the pass once per usable consent and adds the counts up', async () => { + h.consents = { + data: [ + consent('c-new', 'co-1', 0), + consent('c-old', 'co-2', 22), + consent('c-done', 'co-3', 4, 'visma'), + ], + error: null, + } + mockComplete + .mockResolvedValueOnce({ ...EMPTY, candidates: 311, matched: 311, completed: 300, headersUpdated: 300, notHydrated: 11, remaining: 11 }) + .mockResolvedValueOnce({ ...EMPTY, candidates: 384, matched: 384, completed: 384, headersUpdated: 358 }) + .mockResolvedValueOnce({ ...EMPTY }) + + const response = await GET(makeRequest()) + const body = await response.json() + + expect(response.status).toBe(200) + expect(mockComplete).toHaveBeenCalledTimes(3) + expect(mockComplete.mock.calls[0][0]).toMatchObject({ companyId: 'co-1', consentId: 'c-new' }) + expect(mockComplete.mock.calls[0][0].budgetMs).toBeGreaterThan(0) + expect(mockComplete.mock.calls[0][0].budgetMs).toBeLessThanOrEqual(120_000) + // The company with nothing to complete is not counted as worked on. + expect(body.data).toMatchObject({ + consents: 3, + consentsStale: 0, + consentsFailed: 0, + companies: 2, + candidates: 695, + completed: 684, + headersUpdated: 658, + remaining: 11, + notHydrated: 11, + skippedForBudget: 0, + }) + }) + + it('isolates a failing consent: the others still run and the failure is counted', async () => { + h.consents = { + data: [consent('c-revoked', 'co-1', 1), consent('c-live', 'co-2', 2)], + error: null, + } + mockComplete + .mockRejectedValueOnce(new Error('Token refresh failed for fortnox; the connection must be re-authorized')) + .mockResolvedValueOnce({ ...EMPTY, candidates: 5, matched: 5, completed: 5, headersUpdated: 5 }) + + const response = await GET(makeRequest()) + const body = await response.json() + + expect(response.status).toBe(200) + expect(mockComplete).toHaveBeenCalledTimes(2) + expect(body.data).toMatchObject({ consents: 2, consentsFailed: 1, companies: 1, completed: 5 }) + }) + + it('skips consents whose credentials can no longer be refreshed, by token state not consent age', async () => { + // Fortnox refresh tokens live 45 days and rotate on every refresh: a pair + // whose access token expired 46 days ago is dead however young the consent + // row is. A pair refreshed yesterday on a consent from months ago is live. + // A token without an expiry (Bokio) never goes stale. + h.consents = { + data: [ + { ...consent('c-dead', 'co-1', 46), created_at: '2026-09-01T00:00:00Z' }, + { ...consent('c-live', 'co-2', 1), created_at: '2026-04-01T00:00:00Z' }, + consent('c-bokio', 'co-3', null, 'bokio'), + { ...consent('c-no-token', 'co-4', 1), provider_consent_tokens: null }, + ], + error: null, + } + mockComplete.mockResolvedValue({ ...EMPTY }) + + const response = await GET(makeRequest()) + const body = await response.json() + + expect(response.status).toBe(200) + expect(mockComplete.mock.calls.map((c) => c[0].consentId)).toEqual(['c-live', 'c-bokio']) + expect(body.data).toMatchObject({ consents: 2, consentsStale: 2 }) + }) + + it('does not page: every usable consent is visited, not only the newest few', async () => { + // 57 consents were accepted in the last 60 days on prod (2026-09-05); a + // fixed page of the newest ones would leave older companies with row-less + // invoices waiting forever behind companies that are already done. + h.consents = { + data: Array.from({ length: 80 }, (_, i) => consent(`c-${i}`, `co-${i}`, 1)), + error: null, + } + mockComplete.mockResolvedValue({ ...EMPTY }) + + const response = await GET(makeRequest()) + const body = await response.json() + + expect(response.status).toBe(200) + expect(mockComplete).toHaveBeenCalledTimes(80) + expect(body.data).toMatchObject({ consents: 80, skippedForBudget: 0 }) + }) + + it('consentIsUsable reads the token row, tolerating either embed cardinality', () => { + const now = Date.now() + const array = { ...consent('c', 'co', 1), provider_consent_tokens: [{ token_expires_at: new Date(now - DAY_MS).toISOString() }] } + const stale = { ...consent('c', 'co', 1), provider_consent_tokens: [{ token_expires_at: new Date(now - 50 * DAY_MS).toISOString() }] } + const garbage = { ...consent('c', 'co', 1), provider_consent_tokens: { token_expires_at: 'not a date' } } + expect(consentIsUsable(array, now)).toBe(true) + expect(consentIsUsable(stale, now)).toBe(false) + expect(consentIsUsable(garbage, now)).toBe(false) + expect(consentIsUsable({ ...consent('c', 'co', 1), provider_consent_tokens: [] }, now)).toBe(false) + }) + + it('surfaces a failed consent lookup instead of reporting an empty run', async () => { + h.consents = { data: null, error: { message: 'relation does not exist' } } + + const response = await GET(makeRequest()) + + expect(response.status).toBeGreaterThanOrEqual(500) + expect(mockComplete).not.toHaveBeenCalled() + }) +}) diff --git a/app/api/extensions/arcim-migration/complete-invoice-lines/cron/route.ts b/app/api/extensions/arcim-migration/complete-invoice-lines/cron/route.ts new file mode 100644 index 00000000..e4f9d0f0 --- /dev/null +++ b/app/api/extensions/arcim-migration/complete-invoice-lines/cron/route.ts @@ -0,0 +1,172 @@ +import { NextResponse } from 'next/server' +import { loadExtensions } from '@/lib/extensions/loader' +import { extensionRegistry } from '@/lib/extensions/registry' +import { withCronContext } from '@/lib/api/with-cron-context' +import { createServiceClientNoCookies } from '@/lib/auth/api-keys' +import { + completeMigratedInvoiceLines, + type CompleteInvoiceLinesResult, +} from '@/extensions/general/arcim-migration/lib/complete-invoice-lines' + +/** + * GET /api/extensions/arcim-migration/complete-invoice-lines/cron: fetch the + * rows (and the VAT split) for migrated sales invoices that were imported + * without them. + * + * The migration hydrates the provider's detail form inside a fixed budget + * and reports the shortfall; this is what picks the shortfall up. Every run + * walks the accepted consents whose credentials can still be used, newest + * first, and for each company completes as many of its row-less invoices as + * its share of the run allows. A company with nothing left costs one query + * and no provider call (the pass checks our side before it touches the + * consent), so walking every live consent is cheap and no company waits + * behind a fixed page of newer ones. Scheduled hourly in vercel.json (and + * the Docker crontabs); a company the size of Clearstoq (1 125 invoices) is + * done after two or three runs. + * + * "Can still be used" is read off the token row, not the consent's age: + * Fortnox issues a new refresh token on every refresh and each one lives 45 + * days, so a pair whose access token expired more than 45 days ago has not + * been refreshed since and its refresh token is dead. Trying such a consent + * every hour would only log the same PROVIDER_AUTH_EXPIRED; when the company + * reconnects through the wizard, the token row is renewed and the next run + * finds it here. A token without an expiry (Bokio's private tokens) is + * always eligible. + */ + +export const maxDuration = 300 + +/** Leave the function a margin for the DB writes after the last fetch. */ +const RUN_BUDGET_MS = 240_000 +/** One company's share of provider detail fetches per run. */ +const PER_COMPANY_BUDGET_MS = 120_000 +/** Below this the remaining companies wait for the next run. */ +const MIN_COMPANY_BUDGET_MS = 20_000 +/** + * A token pair not refreshed for this long cannot be refreshed any more + * (Fortnox: refresh tokens live 45 days and rotate on every refresh). + */ +const TOKEN_STALE_DAYS = 45 +/** Hard safety on the consent scan; prod holds ~120 accepted consents in total. */ +const MAX_CONSENTS_SCANNED = 500 + +interface ConsentRow { + id: string + company_id: string + provider: string | null + created_at: string + provider_consent_tokens: { token_expires_at: string | null } | { token_expires_at: string | null }[] | null +} + +/** The consent's token row, whichever cardinality PostgREST rendered it with. */ +function tokenOf(consent: ConsentRow): { token_expires_at: string | null } | null { + const tokens = consent.provider_consent_tokens + if (!tokens) return null + return Array.isArray(tokens) ? (tokens[0] ?? null) : tokens +} + +/** Does this consent still hold credentials a run can use? */ +export function consentIsUsable(consent: ConsentRow, now: number): boolean { + const token = tokenOf(consent) + if (!token) return false + if (token.token_expires_at === null) return true + const expiredAt = Date.parse(token.token_expires_at) + if (Number.isNaN(expiredAt)) return false + return now - expiredAt <= TOKEN_STALE_DAYS * 24 * 60 * 60 * 1000 +} + +export const GET = withCronContext('cron.arcim_migration_complete_invoice_lines', async (_request, ctx) => { + loadExtensions() + + // Physical routes under app/api/extensions// compile into every build; + // the registry (generated from extensions.config.json) is what switches an + // extension on. A scheduled-but-disabled cron must fail visibly. + if (!extensionRegistry.get('arcim-migration')) { + ctx.log.warn('arcim-migration extension is not enabled; cron refused') + return NextResponse.json( + { error: 'Migration extension is not enabled', code: 'EXTENSION_DISABLED' }, + { status: 503 }, + ) + } + + const supabase = createServiceClientNoCookies() + + const { data, error } = await supabase + .from('provider_consents') + .select('id, company_id, provider, created_at, provider_consent_tokens(token_expires_at)') + .eq('status', 1) + .not('provider', 'is', null) + .order('created_at', { ascending: false }) + .limit(MAX_CONSENTS_SCANNED) + + if (error) { + throw new Error(`provider_consents lookup failed: ${error.message}`) + } + + const now = Date.now() + const scanned = (data ?? []) as ConsentRow[] + const consents = scanned.filter((consent) => consentIsUsable(consent, now)) + const deadline = now + RUN_BUDGET_MS + let skippedForBudget = 0 + const totals = { + companies: 0, + candidates: 0, + completed: 0, + headersUpdated: 0, + remaining: 0, + notHydrated: 0, + totalMismatch: 0, + failed: 0, + } + + const summary = await ctx.forEach('consent', consents, async (consent, itemCtx) => { + const budgetMs = Math.min(PER_COMPANY_BUDGET_MS, deadline - Date.now()) + if (budgetMs < MIN_COMPANY_BUDGET_MS) { + skippedForBudget++ + return + } + + const result: CompleteInvoiceLinesResult = await completeMigratedInvoiceLines({ + supabase, + companyId: consent.company_id, + consentId: consent.id, + budgetMs, + }) + + if (result.candidates > 0) { + totals.companies++ + totals.candidates += result.candidates + totals.completed += result.completed + totals.headersUpdated += result.headersUpdated + totals.remaining += result.remaining + totals.notHydrated += result.notHydrated + totals.totalMismatch += result.totalMismatch + totals.failed += result.failed + itemCtx.log.info('migrated invoice rows completed for company', { + companyId: consent.company_id, + provider: consent.provider, + candidates: result.candidates, + matched: result.matched, + completed: result.completed, + remaining: result.remaining, + notHydrated: result.notHydrated, + totalMismatch: result.totalMismatch, + hydration: result.hydration, + }) + } + }) + + ctx.log.info('complete-invoice-lines run finished', { + ...totals, skippedForBudget, consents: summary.total, consentsStale: scanned.length - consents.length, + }) + + return NextResponse.json({ + data: { + consents: summary.total, + consentsStale: scanned.length - consents.length, + consentsFailed: summary.failed, + skippedForBudget, + ...totals, + }, + }) +}) diff --git a/components/extensions/general/ArcimMigrationWorkspace.tsx b/components/extensions/general/ArcimMigrationWorkspace.tsx index 9fbc0e51..9b4dab2a 100644 --- a/components/extensions/general/ArcimMigrationWorkspace.tsx +++ b/components/extensions/general/ArcimMigrationWorkspace.tsx @@ -1942,6 +1942,21 @@ function ResultStep({ failed: entityRowStatus(results.salesInvoices.imported, results.salesInvoices.skipReasons) === 'error', }) } + const lineHydration = results.salesInvoices?.hydration + const linesMissing = lineHydration ? lineHydration.needed - lineHydration.hydrated : 0 + if (lineHydration && linesMissing > 0) { + // The detail fetch that carries the rows and the VAT split runs inside + // a fixed budget, open invoices first. Whatever it did not reach was + // imported as a header with a total and no rows; an hourly pass fills + // those in afterwards. Say so here, or the user finds out on the + // invoice page ("fakturorna finns med en total men utan rader"). + entityLines.push({ + label: t('ext_arcim_invoice_lines_label'), + value: t('ext_arcim_invoice_lines_value', { hydrated: lineHydration.hydrated, needed: lineHydration.needed }), + detail: t('ext_arcim_invoice_lines_pending_detail', { count: linesMissing }), + failed: false, + }) + } if (results.salesInvoices?.creditNotesUnlinked) { // Credit notes land as ordinary invoice rows with reversed amounts. The // pairing to the invoice they credit cannot be resolved at import time: diff --git a/docker/crontab.hosted b/docker/crontab.hosted index c8796b0c..c80eeb7c 100644 --- a/docker/crontab.hosted +++ b/docker/crontab.hosted @@ -47,6 +47,7 @@ 15 4 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/whatsapp-inbox/retention/cron */2 * * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/invoice-inbox/sweep/cron 45 5 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/invoice-inbox/underlag-reconcile/cron +20 * * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/arcim-migration/complete-invoice-lines/cron 15 5 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/bookkeeping/accruals/post-due/cron 30 5 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/receipt-hunt/cron 45 5 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/notifications/bookkeeping-digest/cron diff --git a/docker/crontab.self-hosted b/docker/crontab.self-hosted index bb2fc661..3947813f 100644 --- a/docker/crontab.self-hosted +++ b/docker/crontab.self-hosted @@ -47,6 +47,7 @@ 15 4 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/whatsapp-inbox/retention/cron */2 * * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/invoice-inbox/sweep/cron 45 5 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/invoice-inbox/underlag-reconcile/cron +20 * * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/arcim-migration/complete-invoice-lines/cron 15 5 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/bookkeeping/accruals/post-due/cron 30 5 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/receipt-hunt/cron 45 5 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/notifications/bookkeeping-digest/cron diff --git a/extensions/general/arcim-migration/lib/__tests__/complete-invoice-lines.test.ts b/extensions/general/arcim-migration/lib/__tests__/complete-invoice-lines.test.ts new file mode 100644 index 00000000..fd9fae01 --- /dev/null +++ b/extensions/general/arcim-migration/lib/__tests__/complete-invoice-lines.test.ts @@ -0,0 +1,370 @@ +import { describe, it, expect, beforeEach, vi, type Mock } from 'vitest' +import type { SupabaseClient } from '@supabase/supabase-js' +import type { SalesInvoiceDto } from '@/lib/providers/dto' + +/** + * The follow-up pass for migrated sales invoices imported without rows + * (Fortnox, Briox and Björn Lundén ship none in a list payload, and the + * migration's detail hydration is budget-bounded). It must start from OUR + * row-less invoices, join strictly, hydrate only that subset, write rows only + * when the provider's total agrees with the stored one, and leave anything it + * could not reach for the next run rather than guessing. + */ + +vi.mock('@/lib/providers/resolve-consent', () => ({ + resolveConsent: vi.fn().mockResolvedValue({ + consent: { provider: 'fortnox' }, + accessToken: 'tok', + providerCompanyId: undefined, + }), +})) + +vi.mock('@/lib/providers/provider-data-fetcher', () => ({ + fetchSalesInvoicesDirect: vi.fn(), + hydrateSalesInvoices: vi.fn(), +})) + +vi.mock('@/lib/supabase/fetch-all', () => ({ fetchAllRows: vi.fn() })) + +import { resolveConsent } from '@/lib/providers/resolve-consent' +import { fetchAllRows } from '@/lib/supabase/fetch-all' +import { fetchSalesInvoicesDirect, hydrateSalesInvoices } from '@/lib/providers/provider-data-fetcher' +import { completeMigratedInvoiceLines } from '../complete-invoice-lines' + +const mResolve = resolveConsent as Mock +const mFetchAll = fetchAllRows as Mock +const mList = fetchSalesInvoicesDirect as Mock +const mHydrate = hydrateSalesInvoices as Mock + +const HYDRATION = { needed: 0, hydrated: 0, failed: 0, skippedForBudget: 0 } + +const amount = (value: number, currencyCode = 'SEK') => ({ value, currencyCode }) + +/** A hydrated Fortnox-shaped invoice: 1 000 kr net, 25 % VAT, 1 250 kr total. */ +function providerInvoice(overrides: Partial = {}): SalesInvoiceDto { + return { + id: '1001', + invoiceNumber: '1001', + issueDate: '2026-03-14', + currencyCode: 'SEK', + status: 'paid', + supplier: { name: 'Profilio Sweden AB', identifications: [] }, + customer: { name: 'Kund AB', identifications: [] }, + lines: [ + { + id: '1', + description: 'Konsulttid', + quantity: 10, + unitCode: 'h', + unitPrice: amount(100), + lineExtensionAmount: amount(1000), + taxPercent: 25, + }, + ], + taxTotal: { taxAmount: amount(250) }, + legalMonetaryTotal: { + lineExtensionAmount: amount(1000), + taxInclusiveAmount: amount(1250), + payableAmount: amount(1250), + }, + paymentStatus: { paid: true, balance: amount(0) }, + ...overrides, + } +} + +/** A stored row as the pre-#1745 import left it: 25 % label, 0 kr VAT, no rows. */ +function storedRow(overrides: Record = {}) { + return { + id: 'inv-1', + user_id: 'user-1', + customer_id: 'cust-1', + invoice_number: '1001', + invoice_date: '2026-03-14', + total: 1250, + subtotal: 1250, + vat_amount: 0, + vat_rate: 25, + currency: 'SEK', + exchange_rate: null, + invoice_items: [], + ...overrides, + } +} + +/** Every hydrated as given, in order, nothing left unhydrated. */ +function hydratedAll(invoices: SalesInvoiceDto[], unhydratedIds: string[] = []) { + return { + invoices, + hydration: { ...HYDRATION, needed: invoices.length, hydrated: invoices.length - unhydratedIds.length }, + unhydratedIds: new Set(unhydratedIds), + } +} + +interface Call { table: string; method: string; args: unknown[] } + +/** + * Thenable query-builder stand-in. Records every call; resolves with what + * `respond` returns for the table and the methods used on the chain. + */ +function makeSupabase(respond: (table: string, methods: string[], calls: Call[]) => unknown) { + const calls: Call[] = [] + const from = vi.fn((table: string) => { + const chain: Call[] = [] + const builder: Record = {} + for (const method of ['select', 'in', 'insert', 'update', 'eq', 'neq', 'order', 'range']) { + builder[method] = (...args: unknown[]) => { + const call = { table, method, args } + chain.push(call) + calls.push(call) + return builder + } + } + builder.then = (resolve: (v: unknown) => void, reject: (e: unknown) => void) => + Promise.resolve() + .then(() => respond(table, chain.map((c) => c.method), chain)) + .then(resolve, reject) + return builder + }) + return { supabase: { from } as unknown as SupabaseClient, calls } +} + +const ok = { data: [], error: null } + +function insertedRows(calls: Call[]): Record[] { + return calls + .filter((c) => c.table === 'invoice_items' && c.method === 'insert') + .flatMap((c) => c.args[0] as Record[]) +} + +function headerUpdates(calls: Call[]): Record[] { + return calls + .filter((c) => c.table === 'invoices' && c.method === 'update') + .map((c) => c.args[0] as Record) +} + +describe('completeMigratedInvoiceLines', () => { + beforeEach(() => { + vi.clearAllMocks() + mResolve.mockResolvedValue({ consent: { provider: 'fortnox' }, accessToken: 'tok', providerCompanyId: undefined }) + }) + + it('writes the rows and fills a header that held no VAT evidence', async () => { + mFetchAll.mockResolvedValue([storedRow()]) + const dto = providerInvoice() + mList.mockResolvedValue([dto]) + mHydrate.mockResolvedValue(hydratedAll([dto])) + const { supabase, calls } = makeSupabase(() => ok) + + const result = await completeMigratedInvoiceLines({ supabase, companyId: 'co-1', consentId: 'c-1' }) + + expect(result).toMatchObject({ + candidates: 1, providerInvoices: 1, matched: 1, unmatched: 0, + completed: 1, headersUpdated: 1, remaining: 0, totalMismatch: 0, notHydrated: 0, failed: 0, + }) + // Only the matched subset is hydrated, so the budget is never spent on + // invoices already complete on our side. + expect(mHydrate).toHaveBeenCalledTimes(1) + expect(mHydrate.mock.calls[0][3]).toEqual([dto]) + + const rows = insertedRows(calls) + expect(rows).toHaveLength(1) + expect(rows[0]).toMatchObject({ + invoice_id: 'inv-1', + sort_order: 1, + description: 'Konsulttid', + quantity: 10, + unit: 'h', + unit_price: 100, + line_total: 1000, + vat_rate: 25, + vat_amount: 250, + }) + + const headers = headerUpdates(calls) + expect(headers).toHaveLength(1) + expect(headers[0]).toEqual({ + subtotal: 1000, + subtotal_sek: 1000, + vat_amount: 250, + vat_amount_sek: 250, + vat_rate: 25, + vat_treatment: 'standard_25', + }) + // Scoped to the company as well as the id: defense in depth on a + // service-role client. + const update = calls.find((c) => c.table === 'invoices' && c.method === 'update')! + const scope = calls.filter((c) => c.table === 'invoices' && c.method === 'eq' && calls.indexOf(c) > calls.indexOf(update)) + expect(scope.map((c) => c.args)).toEqual([['id', 'inv-1'], ['company_id', 'co-1']]) + // Never the total, status or payments. + expect(Object.keys(headers[0])).not.toContain('total') + expect(Object.keys(headers[0])).not.toContain('status') + }) + + it('writes the rows but leaves a header whose split is consistent (momsfri)', async () => { + mFetchAll.mockResolvedValue([storedRow({ vat_rate: 0, vat_amount: 0, subtotal: 1250 })]) + const dto = providerInvoice() + mList.mockResolvedValue([dto]) + mHydrate.mockResolvedValue(hydratedAll([dto])) + const { supabase, calls } = makeSupabase(() => ok) + + const result = await completeMigratedInvoiceLines({ supabase, companyId: 'co-1', consentId: 'c-1' }) + + expect(result).toMatchObject({ completed: 1, headersUpdated: 0 }) + expect(insertedRows(calls)).toHaveLength(1) + expect(headerUpdates(calls)).toHaveLength(0) + }) + + it('fills a header whose rate is null (the post-#1745 "source did not say")', async () => { + mFetchAll.mockResolvedValue([storedRow({ vat_rate: null, vat_amount: 0, subtotal: 1250 })]) + const dto = providerInvoice() + mList.mockResolvedValue([dto]) + mHydrate.mockResolvedValue(hydratedAll([dto])) + const { supabase, calls } = makeSupabase(() => ok) + + const result = await completeMigratedInvoiceLines({ supabase, companyId: 'co-1', consentId: 'c-1' }) + + expect(result).toMatchObject({ completed: 1, headersUpdated: 1 }) + expect(headerUpdates(calls)[0]).toMatchObject({ vat_rate: 25, vat_amount: 250, subtotal: 1000 }) + }) + + it('derives the SEK twins from the rate the row already carries', async () => { + mFetchAll.mockResolvedValue([storedRow({ currency: 'EUR', exchange_rate: 11.2, total: 1250 })]) + const dto = providerInvoice({ currencyCode: 'EUR' }) + mList.mockResolvedValue([dto]) + mHydrate.mockResolvedValue(hydratedAll([dto])) + const { supabase, calls } = makeSupabase(() => ok) + + await completeMigratedInvoiceLines({ supabase, companyId: 'co-1', consentId: 'c-1' }) + + expect(headerUpdates(calls)[0]).toMatchObject({ subtotal: 1000, subtotal_sek: 11200, vat_amount: 250, vat_amount_sek: 2800 }) + }) + + it('leaves an invoice untouched when the provider total differs from the stored one', async () => { + mFetchAll.mockResolvedValue([storedRow({ total: 1300, subtotal: 1300 })]) + const dto = providerInvoice() + mList.mockResolvedValue([dto]) + mHydrate.mockResolvedValue(hydratedAll([dto])) + const { supabase, calls } = makeSupabase(() => ok) + + const result = await completeMigratedInvoiceLines({ supabase, companyId: 'co-1', consentId: 'c-1' }) + + expect(result).toMatchObject({ matched: 1, completed: 0, totalMismatch: 1, remaining: 1 }) + expect(insertedRows(calls)).toHaveLength(0) + expect(headerUpdates(calls)).toHaveLength(0) + }) + + it('reverses the rows of a kreditfaktura the way the migration does', async () => { + mFetchAll.mockResolvedValue([storedRow({ total: -1250, subtotal: -1250 })]) + const dto = providerInvoice({ invoiceTypeCode: '381', status: 'credited' }) + mList.mockResolvedValue([dto]) + mHydrate.mockResolvedValue(hydratedAll([dto])) + const { supabase, calls } = makeSupabase(() => ok) + + const result = await completeMigratedInvoiceLines({ supabase, companyId: 'co-1', consentId: 'c-1' }) + + expect(result).toMatchObject({ completed: 1, totalMismatch: 0 }) + expect(insertedRows(calls)[0]).toMatchObject({ quantity: -10, unit_price: 100, line_total: -1000, vat_amount: -250 }) + expect(headerUpdates(calls)[0]).toMatchObject({ subtotal: -1000, vat_amount: -250 }) + }) + + it('leaves invoices the budget did not reach for the next run', async () => { + mFetchAll.mockResolvedValue([storedRow(), storedRow({ id: 'inv-2', invoice_number: '1002' })]) + const first = providerInvoice() + const second = providerInvoice({ id: '1002', invoiceNumber: '1002', lines: [] }) + mList.mockResolvedValue([first, second]) + // The second came back in list form only (no rows): the budget ran out. + mHydrate.mockResolvedValue(hydratedAll([first, second], ['1002'])) + const { supabase, calls } = makeSupabase(() => ok) + + const result = await completeMigratedInvoiceLines({ supabase, companyId: 'co-1', consentId: 'c-1' }) + + expect(result).toMatchObject({ candidates: 2, matched: 2, completed: 1, notHydrated: 1, remaining: 1 }) + expect(insertedRows(calls).map((r) => r.invoice_id)).toEqual(['inv-1']) + }) + + it('does not join an ambiguous key, and does not hydrate when nothing joined', async () => { + // Two stored rows with the same number and date: a wrong join would put + // one invoice's rows under the other. + mFetchAll.mockResolvedValue([storedRow(), storedRow({ id: 'inv-dup' })]) + mList.mockResolvedValue([providerInvoice()]) + const { supabase, calls } = makeSupabase(() => ok) + + const result = await completeMigratedInvoiceLines({ supabase, companyId: 'co-1', consentId: 'c-1' }) + + expect(result).toMatchObject({ candidates: 2, matched: 0, unmatched: 2, completed: 0, remaining: 2 }) + expect(mHydrate).not.toHaveBeenCalled() + expect(insertedRows(calls)).toHaveLength(0) + }) + + it('costs one query and no provider call when the company has nothing to complete', async () => { + mFetchAll.mockResolvedValue([storedRow({ invoice_items: [{ id: 'item-1' }] })]) + const { supabase, calls } = makeSupabase(() => ok) + + const result = await completeMigratedInvoiceLines({ supabase, companyId: 'co-1', consentId: 'c-1' }) + + expect(result).toMatchObject({ candidates: 0, completed: 0, remaining: 0 }) + expect(mResolve).not.toHaveBeenCalled() + expect(mList).not.toHaveBeenCalled() + expect(calls).toHaveLength(0) + }) + + it('skips an invoice that gained rows since the candidates were loaded', async () => { + mFetchAll.mockResolvedValue([storedRow()]) + const dto = providerInvoice() + mList.mockResolvedValue([dto]) + mHydrate.mockResolvedValue(hydratedAll([dto])) + const { supabase, calls } = makeSupabase((table, methods) => + table === 'invoice_items' && methods.includes('in') ? { data: [{ invoice_id: 'inv-1' }], error: null } : ok, + ) + + const result = await completeMigratedInvoiceLines({ supabase, companyId: 'co-1', consentId: 'c-1' }) + + expect(result).toMatchObject({ completed: 0, failed: 0, remaining: 1 }) + expect(insertedRows(calls)).toHaveLength(0) + expect(headerUpdates(calls)).toHaveLength(0) + }) + + it('retries per invoice when the batch insert fails, and counts the offender', async () => { + mFetchAll.mockResolvedValue([storedRow(), storedRow({ id: 'inv-2', invoice_number: '1002' })]) + const first = providerInvoice() + const second = providerInvoice({ id: '1002', invoiceNumber: '1002' }) + mList.mockResolvedValue([first, second]) + mHydrate.mockResolvedValue(hydratedAll([first, second])) + const { supabase, calls } = makeSupabase((table, methods, chain) => { + if (table !== 'invoice_items' || !methods.includes('insert')) return ok + const rows = chain[0].args[0] as { invoice_id: string }[] + if (rows.length > 1) return { data: null, error: { message: 'batch rejected' } } + return rows[0].invoice_id === 'inv-2' ? { data: null, error: { message: 'check violation' } } : ok + }) + + const result = await completeMigratedInvoiceLines({ supabase, companyId: 'co-1', consentId: 'c-1' }) + + expect(result).toMatchObject({ completed: 1, failed: 1, headersUpdated: 1, remaining: 1 }) + expect(headerUpdates(calls)).toHaveLength(1) + }) + + it('dry run: reports the plan and writes nothing', async () => { + mFetchAll.mockResolvedValue([storedRow()]) + const dto = providerInvoice() + mList.mockResolvedValue([dto]) + mHydrate.mockResolvedValue(hydratedAll([dto])) + const { supabase, calls } = makeSupabase(() => ok) + + const result = await completeMigratedInvoiceLines({ supabase, companyId: 'co-1', consentId: 'c-1', dryRun: true }) + + expect(result).toMatchObject({ dryRun: true, completed: 1, headersUpdated: 1, remaining: 0 }) + expect(calls).toHaveLength(0) + }) + + it('passes the budget through to the hydration', async () => { + mFetchAll.mockResolvedValue([storedRow()]) + const dto = providerInvoice() + mList.mockResolvedValue([dto]) + mHydrate.mockResolvedValue(hydratedAll([dto])) + const { supabase } = makeSupabase(() => ok) + + await completeMigratedInvoiceLines({ supabase, companyId: 'co-1', consentId: 'c-1', budgetMs: 45_000 }) + + expect(mHydrate.mock.calls[0][4]).toBe(45_000) + }) +}) diff --git a/extensions/general/arcim-migration/lib/complete-invoice-lines.ts b/extensions/general/arcim-migration/lib/complete-invoice-lines.ts new file mode 100644 index 00000000..5f3c6c34 --- /dev/null +++ b/extensions/general/arcim-migration/lib/complete-invoice-lines.ts @@ -0,0 +1,409 @@ +/** + * Finish a migrated sales-invoice register: fetch the rows (and the VAT split) + * for the invoices that were imported without them. + * + * The migration maps invoices from the provider's LIST payload and hydrates + * the detail form inside a fixed budget (see lib/providers/provider-data- + * fetcher.ts). Fortnox, Briox and Björn Lundén ship no rows in a list + * response, so every invoice the budget did not reach lands as a header with + * a total and no invoice_items behind it: Profilio (384 of 384, migrated + * before hydration existed), Loftux (311 of 672), Damac (182 of 542), + * Clearstoq (1 125 of 1 125). Nothing ever came back for them, and the wizard + * had not said so. + * + * This pass is the follow-up. It is re-runnable and makes progress on every + * run: it starts from OUR side (the non-draft invoices in this company that + * have no rows), joins them to the provider's register on invoice number AND + * date (unique on both sides, the same strictness as the registration-voucher + * relink), hydrates only those, and writes each invoice's rows once the + * provider's detail total agrees with the stored total to the öre. A run that + * runs out of budget leaves the rest for the next one; nothing is guessed and + * nothing is inserted for an invoice the provider does not know. + * + * What it writes, and only this: + * - invoice_items for an invoice that has none; + * - the header VAT split (subtotal, vat_amount, vat_rate, vat_treatment and + * their SEK twins) when the stored split is the "unknown" shape the old + * import left behind (no rate, or 0 kr VAT beside subtotal = total). + * Never the total, the status, the customer, payments or any journal entry: + * momsdeklaration and every report read the ledger, not these columns, so + * filling them changes what the invoice page shows and nothing that was + * filed (see the 2026-08-22 verification in DECISIONS.md). + */ + +import type { SupabaseClient } from '@supabase/supabase-js' +import { ISO_DATE_RE } from '@/lib/invariants' +import { createLogger } from '@/lib/logger' +import { equalOre, roundOre } from '@/lib/money' +import { chunk } from '@/lib/utils' +import type { SalesInvoiceDto } from '@/lib/providers/dto' +import type { ProviderName } from '@/lib/providers/types' +import { resolveConsent } from '@/lib/providers/resolve-consent' +import { + fetchSalesInvoicesDirect, + hydrateSalesInvoices, + type HydrationReport, +} from '@/lib/providers/provider-data-fetcher' +import { fetchAllRows } from '@/lib/supabase/fetch-all' +import { mapSalesInvoice } from './entity-mapper' + +const log = createLogger('extensions/arcim-migration/complete-invoice-lines') + +export interface CompleteInvoiceLinesOptions { + supabase: SupabaseClient + companyId: string + consentId: string + /** Report the plan without writing. */ + dryRun?: boolean + /** Wall-clock ceiling for the provider detail fetches, in ms. */ + budgetMs?: number +} + +export interface CompleteInvoiceLinesResult { + /** Non-draft invoices in this company that carry no rows. */ + candidates: number + /** Sales invoices the provider's register lists. */ + providerInvoices: number + /** Candidates that joined to exactly one provider invoice (number + date). */ + matched: number + /** Candidates with no unique provider counterpart: native rows, ambiguous keys, or gone at the provider. */ + unmatched: number + /** Invoices whose rows were written this run (or would be, on a dry run). */ + completed: number + /** Of `completed`, those whose header VAT split was rewritten from the detail form. */ + headersUpdated: number + /** Matched and hydrated, but the provider's total differs from the stored one; left untouched. */ + totalMismatch: number + /** Matched and hydrated, but the detail form itself carries no rows. */ + noLinesAtProvider: number + /** Matched but not hydrated this run (budget, auth, or a failed fetch); the next run retries them. */ + notHydrated: number + /** Rows written but the header left alone because the detail form established no VAT. */ + vatUnresolved: number + /** Invoices whose write failed at the database. */ + failed: number + /** Candidates still without rows after this run: `candidates - completed`. */ + remaining: number + hydration: HydrationReport + dryRun: boolean +} + +interface CandidateRow { + id: string + user_id: string + customer_id: string + invoice_number: string | null + invoice_date: string + total: number + subtotal: number | null + vat_amount: number | null + vat_rate: number | null + currency: string | null + exchange_rate: number | null + invoice_items: { id: string }[] | null +} + +/** Invoices per statement. Small enough that a chunk's rows stay one request. */ +const WRITE_CHUNK_SIZE = 100 + +/** + * "number::YYYY-MM-DD", the same key the registration-voucher relink joins + * on. A date that does not start like an ISO date joins nothing. + */ +function joinKey(number: string | null | undefined, date: string | null | undefined): string | null { + if (!number || !date) return null + const day = date.slice(0, 10) + return ISO_DATE_RE.test(day) ? `${number}::${day}` : null +} + +/** A map that forgets keys seen more than once, so those are never joined on. */ +function uniqueByKey(rows: readonly T[], keyOf: (row: T) => string | null): Map { + const out = new Map() + const dupes = new Set() + for (const row of rows) { + const key = keyOf(row) + if (!key) continue + if (out.has(key) || dupes.has(key)) { + out.delete(key) + dupes.add(key) + continue + } + out.set(key, row) + } + return out +} + + +/** + * Is the stored VAT split the shape the old import left behind? + * + * A null rate is the post-#1745 "the source did not say". A non-zero rate + * label beside 0 kr of VAT and subtotal = total is what the pre-#1745 mapper + * wrote for every Fortnox invoice (25 % on top of nothing): a contradiction, + * not a fact. Both mean the columns hold no evidence, so the detail form may + * overwrite them. A split that is consistent with itself, including a + * genuinely momsfri one (rate 0, 0 kr VAT, subtotal = total), is evidence + * and is left as it is; the rows are still written. + */ +function headerHoldsNoVatEvidence(row: CandidateRow): boolean { + if (row.vat_rate === null) return true + if (row.vat_rate === 0) return false + const subtotal = row.subtotal ?? row.total + const vat = row.vat_amount ?? 0 + return equalOre(vat, 0) && equalOre(subtotal, row.total) +} + +/** SEK twin of an amount on this row, or null when the row has no rate. */ +function toRowSek(amount: number, row: CandidateRow): number | null { + if (!row.currency || row.currency.toUpperCase() === 'SEK') return roundOre(amount) + return row.exchange_rate != null ? roundOre(amount * row.exchange_rate) : null +} + +async function loadCandidates(supabase: SupabaseClient, companyId: string): Promise { + const rows = await fetchAllRows(({ from, to }) => + supabase + .from('invoices') + .select( + 'id, user_id, customer_id, invoice_number, invoice_date, total, subtotal, vat_amount, vat_rate, currency, exchange_rate, invoice_items(id)', + ) + .eq('company_id', companyId) + .eq('document_type', 'invoice') + // A draft without rows is a draft someone is still writing, not an + // import shortfall (a migrated row is a draft only when the source had + // no voucher for it anyway). + .neq('status', 'draft') + .order('id', { ascending: true }) + .range(from, to), + ) + return rows.filter((row) => (row.invoice_items?.length ?? 0) === 0) +} + +/** Ids among `ids` that gained rows since the candidates were loaded. */ +async function alreadyFilled(supabase: SupabaseClient, ids: string[]): Promise> { + const { data, error } = await supabase + .from('invoice_items') + .select('invoice_id') + .in('invoice_id', ids) + if (error) throw new Error(`invoice_items lookup failed: ${error.message}`) + return new Set(((data ?? []) as { invoice_id: string }[]).map((r) => r.invoice_id)) +} + +/** The header VAT split the detail form established, ready to write. */ +interface HeaderFill { + subtotal: number + subtotalSek: number | null + vatAmount: number + vatAmountSek: number | null + vatRate: number | null + vatTreatment: string +} + +interface PlannedWrite { + row: CandidateRow + items: Record[] + header: HeaderFill | null +} + +export async function completeMigratedInvoiceLines( + options: CompleteInvoiceLinesOptions, +): Promise { + const { supabase, companyId, consentId, dryRun = false, budgetMs } = options + + const result: CompleteInvoiceLinesResult = { + candidates: 0, + providerInvoices: 0, + matched: 0, + unmatched: 0, + completed: 0, + headersUpdated: 0, + totalMismatch: 0, + noLinesAtProvider: 0, + notHydrated: 0, + vatUnresolved: 0, + failed: 0, + remaining: 0, + hydration: { needed: 0, hydrated: 0, failed: 0, skippedForBudget: 0 }, + dryRun, + } + + // Our side first, and before the consent is touched: a company with nothing + // to complete costs one query and no token refresh at the provider. + const candidates = await loadCandidates(supabase, companyId) + result.candidates = candidates.length + if (candidates.length === 0) return result + + const resolved = await resolveConsent(companyId, consentId) + const provider = resolved.consent.provider as ProviderName + + const listed = await fetchSalesInvoicesDirect(provider, resolved.accessToken, resolved.providerCompanyId) + result.providerInvoices = listed.length + + const providerByKey = uniqueByKey(listed, (dto) => joinKey(dto.invoiceNumber, dto.issueDate)) + const oursByKey = uniqueByKey(candidates, (row) => joinKey(row.invoice_number, row.invoice_date)) + + const pairs: { row: CandidateRow; dto: SalesInvoiceDto }[] = [] + for (const [key, row] of oursByKey) { + const dto = providerByKey.get(key) + if (dto) pairs.push({ row, dto }) + } + result.matched = pairs.length + result.unmatched = candidates.length - pairs.length + + if (pairs.length === 0) { + result.remaining = candidates.length + return result + } + + // Only the matched subset is hydrated, so every run spends its budget on + // invoices that are still incomplete here, never on ones already done. + const hydrated = await hydrateSalesInvoices( + provider, + resolved.accessToken, + resolved.providerCompanyId, + pairs.map((pair) => pair.dto), + budgetMs, + ) + result.hydration = hydrated.hydration + + const planned: PlannedWrite[] = [] + for (let i = 0; i < pairs.length; i++) { + const { row } = pairs[i] + const dto = hydrated.invoices[i] + + if (hydrated.unhydratedIds.has(dto.id)) { + result.notHydrated++ + continue + } + if (dto.lines.length === 0) { + result.noLinesAtProvider++ + continue + } + + // The same mapper the migration used, so a row written here is + // indistinguishable from one written by a fully hydrated import. No FX + // index: the SEK twins are derived from the rate the row already carries. + const mapped = mapSalesInvoice(dto, row.user_id, companyId, row.customer_id) + const mappedTotal = mapped.invoice.total as number + if (!equalOre(mappedTotal, row.total)) { + result.totalMismatch++ + log.warn('detail total differs from the stored total; invoice left untouched', { + companyId, invoiceId: row.id, invoiceNumber: row.invoice_number, stored: row.total, provider: mappedTotal, + }) + continue + } + + let header: HeaderFill | null = null + if (mapped.vatUnresolved) { + result.vatUnresolved++ + } else if (headerHoldsNoVatEvidence(row)) { + const subtotal = mapped.invoice.subtotal as number + const vatAmount = mapped.invoice.vat_amount as number + header = { + subtotal, + subtotalSek: toRowSek(subtotal, row), + vatAmount, + vatAmountSek: toRowSek(vatAmount, row), + vatRate: mapped.invoice.vat_rate as number | null, + vatTreatment: mapped.invoice.vat_treatment as string, + } + } + + planned.push({ + row, + items: mapped.items.map((item) => ({ ...item, invoice_id: row.id })), + header, + }) + } + + if (dryRun) { + result.completed = planned.length + result.headersUpdated = planned.filter((p) => p.header).length + result.remaining = candidates.length - result.completed + return result + } + + for (const batch of chunk(planned, WRITE_CHUNK_SIZE)) { + // A concurrent run (the wizard and the cron, or two crons overlapping) + // may have filled some of these since the candidates were loaded. Rows + // are appended, never replaced, so a second write would double them. + const filled = await alreadyFilled(supabase, batch.map((p) => p.row.id)) + const todo = batch.filter((p) => !filled.has(p.row.id)) + + const written = await insertRows(supabase, todo) + for (const plan of todo) { + if (!written.has(plan.row.id)) { + result.failed++ + continue + } + result.completed++ + if (!plan.header) continue + // Written as a literal so the schema guard checks these columns. + const { error } = await supabase + .from('invoices') + .update({ + subtotal: plan.header.subtotal, + subtotal_sek: plan.header.subtotalSek, + vat_amount: plan.header.vatAmount, + vat_amount_sek: plan.header.vatAmountSek, + vat_rate: plan.header.vatRate, + vat_treatment: plan.header.vatTreatment, + }) + .eq('id', plan.row.id) + .eq('company_id', companyId) + if (error) { + // The rows landed; only the header split is still the old shape. The + // next run will not revisit this invoice (it now has rows), so say so. + log.error('header VAT update failed after the rows were written', { + companyId, invoiceId: plan.row.id, reason: error.message, + }) + continue + } + result.headersUpdated++ + } + } + + result.remaining = candidates.length - result.completed + log.info('migrated invoice rows completed', { + companyId, + candidates: result.candidates, + matched: result.matched, + completed: result.completed, + headersUpdated: result.headersUpdated, + notHydrated: result.notHydrated, + totalMismatch: result.totalMismatch, + failed: result.failed, + remaining: result.remaining, + hydration: result.hydration, + }) + return result +} + +/** + * Insert every plan's rows: one statement for the batch, and on failure one + * statement per invoice so a single bad row rejects its own invoice, not the + * hundred beside it. An invoice's rows never split across statements: it + * either has all of them or none. + */ +async function insertRows(supabase: SupabaseClient, plans: PlannedWrite[]): Promise> { + const written = new Set() + if (plans.length === 0) return written + + const bulk = await supabase.from('invoice_items').insert(plans.flatMap((p) => p.items)) + if (!bulk.error) { + for (const plan of plans) written.add(plan.row.id) + return written + } + + log.warn('bulk invoice_items insert failed; retrying per invoice', { reason: bulk.error.message }) + for (const plan of plans) { + const { error } = await supabase.from('invoice_items').insert(plan.items) + if (error) { + log.error('invoice_items insert failed', { + invoiceId: plan.row.id, invoiceNumber: plan.row.invoice_number, reason: error.message, + }) + continue + } + written.add(plan.row.id) + } + return written +} diff --git a/lib/providers/__tests__/hydrate-invoices.test.ts b/lib/providers/__tests__/hydrate-invoices.test.ts index f8e9e727..7dc001a2 100644 --- a/lib/providers/__tests__/hydrate-invoices.test.ts +++ b/lib/providers/__tests__/hydrate-invoices.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { fetchSalesInvoicesHydrated } from '../provider-data-fetcher'; +import { fetchSalesInvoicesDirect, fetchSalesInvoicesHydrated, hydrateSalesInvoices } from '../provider-data-fetcher'; /** * Hydration is what makes the VAT fix work in production: the list payload @@ -206,3 +206,63 @@ describe('fetchSalesInvoicesHydrated: detail id comes from the configured idFiel expect(invoices[0]?.taxTotal?.taxAmount.value).toBe(250); }); }); + +describe('hydrateSalesInvoices: a caller-chosen subset of an already-listed register', () => { + let requested: string[]; + + beforeEach(() => { + requested = []; + vi.stubEnv('UPSTASH_REDIS_REST_URL', ''); + vi.stubEnv('UPSTASH_REDIS_REST_TOKEN', ''); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + }); + + const json = (body: unknown) => + new Response(JSON.stringify(body), { status: 200, headers: { 'Content-Type': 'application/json' } }); + + it('requests the detail form only for the invoices it was given', async () => { + // A follow-up pass knows which invoices are still incomplete on its own + // side. Re-hydrating the whole register would spend every run on the + // same open invoices first and never reach the rest. + const third = { ...PAID, DocumentNumber: 6, InvoiceDate: '2025-11-03' }; + vi.stubGlobal('fetch', vi.fn(async (url: string) => { + requested.push(url); + if (url.includes('/invoices/5')) return json(detailFor(5, 500)); + if (url.includes('/invoices/6')) return json(detailFor(6, 500)); + return json(listResponse([OPEN, PAID, third])); + })); + + const listed = await fetchSalesInvoicesDirect('fortnox', 'token'); + const subset = listed.filter((dto) => dto.id !== '4'); + + const { invoices, hydration, unhydratedIds } = await hydrateSalesInvoices('fortnox', 'token', undefined, subset); + + expect(hydration).toMatchObject({ needed: 2, hydrated: 2, failed: 0, skippedForBudget: 0 }); + expect(unhydratedIds.size).toBe(0); + // Same order as given, so the caller can pair results back by index. + expect(invoices.map((dto) => dto.id)).toEqual(['5', '6']); + expect(invoices.every((dto) => dto.lines.length === 1)).toBe(true); + const details = requested.filter((u) => /\/invoices\/\d/.test(u)); + expect(details).toHaveLength(2); + expect(details.some((u) => u.includes('/invoices/4'))).toBe(false); + }); + + it('reports the subset it could not reach, in the caller\'s ids', async () => { + vi.stubGlobal('fetch', vi.fn(async (url: string) => { + requested.push(url); + return json(listResponse([OPEN, PAID])); + })); + + const listed = await fetchSalesInvoicesDirect('fortnox', 'token'); + + const { invoices, hydration, unhydratedIds } = await hydrateSalesInvoices('fortnox', 'token', undefined, listed, 0); + + expect(hydration).toMatchObject({ needed: 2, hydrated: 0, skippedForBudget: 2 }); + expect([...unhydratedIds].sort()).toEqual(['4', '5']); + expect(invoices).toHaveLength(2); + }); +}); diff --git a/lib/providers/provider-data-fetcher.ts b/lib/providers/provider-data-fetcher.ts index d25f5ba8..9bf74c1e 100644 --- a/lib/providers/provider-data-fetcher.ts +++ b/lib/providers/provider-data-fetcher.ts @@ -746,7 +746,31 @@ export async function fetchSalesInvoicesHydrated( budgetMs: number = DEFAULT_HYDRATION_BUDGET_MS, ): Promise> { const invoices = await fetchSalesInvoicesDirect(provider, accessToken, providerCompanyId); + return hydrateSalesInvoices(provider, accessToken, providerCompanyId, invoices, budgetMs); +} +/** + * Hydrate a caller-chosen set of already-listed sales invoices. + * + * The migration's own pass (above) spends its budget on the WHOLE register, + * open invoices first, and reports what it did not reach. A follow-up that + * wants to finish the job must not repeat that: re-hydrating the register + * from the top would spend every run on the same open invoices and never get + * to the ones still missing their rows. This entry point takes the subset the + * caller already knows to be incomplete on its own side, so each run makes + * progress on exactly those. Invoices that need nothing (a list payload that + * carried its rows) pass through unrequested, as in the full pass. + * + * Returns the invoices in the order given; see `hydrateInvoices` for the + * report and `unhydratedIds` semantics. + */ +export async function hydrateSalesInvoices( + provider: ProviderName, + accessToken: string, + providerCompanyId: string | undefined, + invoices: SalesInvoiceDto[], + budgetMs: number = DEFAULT_HYDRATION_BUDGET_MS, +): Promise> { const { items, report, unhydratedIds } = await hydrateInvoices( invoices, salesInvoiceNeedsDetail, diff --git a/messages/en.json b/messages/en.json index 62a901f7..5d9b0477 100644 --- a/messages/en.json +++ b/messages/en.json @@ -5601,6 +5601,9 @@ "ext_arcim_registration_links_already_linked": "{count, plural, one {# was already linked earlier} other {# were already linked earlier}}", "ext_arcim_credit_notes_label": "Credit notes", "ext_arcim_credit_notes_unlinked_detail": "{count, plural, one {# credit note was imported without a link to the invoice it credits: your old system sends no reference to the original invoice. Its amounts are reversed and the record is complete, but the link to the original invoice is missing.} other {# credit notes were imported without a link to the invoices they credit: your old system sends no reference to the original invoice. Their amounts are reversed and the records are complete, but the link to the original invoice is missing.}}", + "ext_arcim_invoice_lines_label": "Invoice lines", + "ext_arcim_invoice_lines_value": "{hydrated} of {needed} invoices fetched with their lines", + "ext_arcim_invoice_lines_pending_detail": "{count, plural, one {# invoice was imported with its total but without its lines: your old system only returns the lines in a separate fetch per invoice, and there was not time for all of them. The lines are fetched automatically in the background within the hour.} other {# invoices were imported with their totals but without their lines: your old system only returns the lines in a separate fetch per invoice, and there was not time for all of them. The lines are fetched automatically in the background within the hour.}}", "ext_arcim_migration_description": "Migrate bookkeeping from Fortnox, Visma, Bokio, Björn Lundén or Briox", "ext_arcim_migration_long_description": "Move all bookkeeping data from your old system to accounted. Imports chart of accounts, vouchers, customers, suppliers and open invoices automatically via a secure API integration directly with the provider.", "ext_arcim_bokio_token_description": "Enter the integration token and company ID from Bokio to let {appName} read your bookkeeping data.", diff --git a/messages/sv.json b/messages/sv.json index 6562abe3..16d5e229 100644 --- a/messages/sv.json +++ b/messages/sv.json @@ -5601,6 +5601,9 @@ "ext_arcim_registration_links_already_linked": "{count, plural, one {# var redan länkad sedan tidigare} other {# var redan länkade sedan tidigare}}", "ext_arcim_credit_notes_label": "Kreditfakturor", "ext_arcim_credit_notes_unlinked_detail": "{count, plural, one {# kreditfaktura importerades utan koppling till fakturan den krediterar: ditt gamla system skickar ingen referens till ursprungsfakturan. Beloppen är omvända och posten är komplett, men kopplingen till ursprungsfakturan saknas.} other {# kreditfakturor importerades utan koppling till fakturorna de krediterar: ditt gamla system skickar ingen referens till ursprungsfakturan. Beloppen är omvända och posterna är kompletta, men kopplingen till ursprungsfakturan saknas.}}", + "ext_arcim_invoice_lines_label": "Fakturarader", + "ext_arcim_invoice_lines_value": "{hydrated} av {needed} fakturor hämtade med rader", + "ext_arcim_invoice_lines_pending_detail": "{count, plural, one {# faktura importerades med totalbelopp men utan fakturarader: ditt gamla system lämnar ut raderna först i en separat hämtning per faktura, och tiden räckte inte till alla. Raderna hämtas automatiskt i bakgrunden inom en timme.} other {# fakturor importerades med totalbelopp men utan fakturarader: ditt gamla system lämnar ut raderna först i en separat hämtning per faktura, och tiden räckte inte till alla. Raderna hämtas automatiskt i bakgrunden inom en timme.}}", "ext_arcim_migration_description": "Migrera bokföring från Fortnox, Visma, Bokio, Björn Lundén eller Briox", "ext_arcim_migration_long_description": "Flytta all bokföringsdata från ditt gamla system till accounted. Importerar kontoplan, verifikationer, kunder, leverantörer och öppna fakturor automatiskt via säker API-integration direkt med leverantören.", "ext_arcim_bokio_token_description": "Ange integrationstoken och företags-ID från Bokio för att ge {appName} tillgång att läsa din bokföringsdata.", diff --git a/tests/schema/no-phantom-columns.test.ts b/tests/schema/no-phantom-columns.test.ts index 27249fa0..31ac007b 100644 --- a/tests/schema/no-phantom-columns.test.ts +++ b/tests/schema/no-phantom-columns.test.ts @@ -174,7 +174,14 @@ const KNOWN_STALE_ON_CONFLICT: Record = {} // one place (and held by CHECK supplier_invoices_credit_note_not_payable); // its columns are the object literal in that file, pinned by // credit-note.test.ts. Merged with main (#2288) at 399: 402. -const UNRESOLVED_CEILING = 402 +// 2026-09-04: +2 for the migrated-invoice row completion +// (extensions/general/arcim-migration/lib/complete-invoice-lines.ts): the +// invoice_items rows come from the migration's own mapSalesInvoiceLine, the +// same row array the orchestrator already inserts (counted above), written +// once as a batch and again per invoice when the batch is rejected. The header +// VAT update in the same module is an object literal and is checked. Merged +// with main (#2289) at 402: 404. +const UNRESOLVED_CEILING = 404 /** * Floor on statically resolved column references. Guards the guard: if a change diff --git a/vercel.json b/vercel.json index 7d518e6f..ae84af77 100644 --- a/vercel.json +++ b/vercel.json @@ -98,6 +98,10 @@ "path": "/api/extensions/invoice-inbox/underlag-reconcile/cron", "schedule": "45 5 * * *" }, + { + "path": "/api/extensions/arcim-migration/complete-invoice-lines/cron", + "schedule": "20 * * * *" + }, { "path": "/api/bookkeeping/accruals/post-due/cron", "schedule": "15 5 * * *"