diff --git a/DECISIONS.md b/DECISIONS.md index 87e8aa98..c3b40135 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -1148,3 +1148,4 @@ One line per decision: `[YYYY-MM-DD] : `. Appended by agents and [2026-08-21] MCP signed Storage URLs (gnubok_create_document_upload upload_url, gnubok_get_document_content signed_url, audit-package download_url) are served through a same-origin proxy, /api/storage/[...path] → .supabase.co/storage/v1/object/{sign,upload/sign}/documents/..., instead of a Next rewrite: Claude Desktop's sandbox only reaches the MCP host (app.accounted.se) and blocked the PUT to supabase.co (Fabian, 2026-08-21). A route handler reads NEXT_PUBLIC_SUPABASE_URL at runtime (rewrites bake at build, which breaks the Docker image), is deliberately NOT withRouteContext (the signed token is the only credential, validated by Storage per object path; the proxy forwards only signed documents-bucket paths to our own host) and is a no-op rewrite when NEXT_PUBLIC_APP_URL is unset so a self-host never gets a localhost link. [2026-08-21] RIP-4 cascade step 3 (UI): the AI booking proposal is surfaced INSIDE the existing QuickReviewDialog rather than a new inline-row card, so it reuses that dialog's proven, deterministic, balanced commit path (POST /api/transactions/[id]/categorize) instead of a parallel one. components/transactions/AiCategorizeProposal.tsx fetches POST /api/agent/categorize on dialog open (keyed on tx.id so it remounts per transaction), pre-fills accountOverride + vatTreatment via handleAccountChange (class-2 VAT clearing preserved), and shows the confidence band (säker/trolig/välj konto) + "Varför" + the candidate alternatives (click to re-apply). Falls back SILENTLY to the deterministic defaults on error, and shows a soft note on 503 (ai_unconfigured) — the dialog always works without AI. NO silent auto-posting (founder call, avoids the storno-on-undo mess): "säker" = pre-filled, one-tap Bokför via the dialog's existing button; true hands-off auto-book waits for calibration. i18n: strings inline Swedish for now (assistant surface), lift to messages/{sv,en}.json before final merge. Confidence bands (0.8/0.5) are placeholders until calibration. Needs founder visual sign-off before merge ([[project_nav_ia_redesign]]). [2026-08-21] RIP-4 step 4 = calibration. lib/agent/categorize/calibration.ts is the engine: isotonic regression via pool-adjacent-violators (distribution-free, monotonic) over (confidence, was_correct) samples → a calibrator that turns raw selector confidence into a probability that actually means what it says; plus reliabilityByBucket/ECE and bandFor(). bandFor NEVER returns 'auto' without a fitted calibrator (no silent booking on an unproven score) and never auto-books above an amount cap (default 2000 kr) — so "säker" stays honest until proven. Measurement loop: migration 20260821100000 categorize_calibration_samples (append-only, company-scoped RLS, confidence CHECK [0,1]); POST /api/agent/categorize/outcome logs one sample (proposed vs booked account → was_correct) fire-and-forget from QuickReviewDialog on a successful book (sandbox skipped to keep the corpus clean); AiCategorizeProposal surfaces the proposal metadata via onProposal. scripts/fit-categorize-calibration.ts (READ-ONLY) prints the reliability diagram + ECE + fitted calibrator once data exists — run it in a few weeks, then store the calibrator/thresholds where bandFor reads them and only THEN consider enabling auto-book. Fitting needs >=200 real samples so nothing calibrates today; the loop just starts collecting. Migration applies on merge (auto-apply-on-merge active) — not applied manually. +[2026-08-21] The categorize selector now reads the underlag, not just the bank line — the highest-leverage quality lever for the cold-start majority (prod: 365 companies with 32.7k unbooked tx, median 0 templates, so the LLM carries them). lib/agent/categorize/underlag.ts gathers the matched receipt/invoice text (receipts.matched_transaction_id + invoice_inbox_items.matched_transaction_id + the transaction's own document_attachments), rendered as bounded Swedish text (supplier, date, total, moms, line items). Core reads these tables directly via supabase (table names, not @/extensions imports — the tables live in the shared DB). POST /api/agent/categorize gathers it server-side when the caller didn't pass `underlag`, so the model sees the actual supplier + line items. Best-effort ('' on any failure); server-side only, no client change (so no conflict with the calibration PR #1784 which also touches the dialog). Prod read (project pwxtzglxptnnvjrpixpg) also confirmed: 3342 active counterparty templates / 26k occurrences → established users get strong instant candidates; NO backfill needed (templates already reflect historical bookings). diff --git a/app/api/agent/categorize/__tests__/route.test.ts b/app/api/agent/categorize/__tests__/route.test.ts index 4b142ad1..4a637998 100644 --- a/app/api/agent/categorize/__tests__/route.test.ts +++ b/app/api/agent/categorize/__tests__/route.test.ts @@ -18,6 +18,8 @@ const aiStatus = vi.fn() vi.mock('@/lib/ai', () => ({ getAiStatus: () => aiStatus() })) const gatherCandidates = vi.fn() vi.mock('@/lib/agent/categorize/candidates', () => ({ gatherCandidates: (...a: unknown[]) => gatherCandidates(...a) })) +const gatherUnderlag = vi.fn() +vi.mock('@/lib/agent/categorize/underlag', () => ({ gatherUnderlag: (...a: unknown[]) => gatherUnderlag(...a) })) const selectAccount = vi.fn() vi.mock('@/lib/agent/categorize/select-account', () => ({ selectAccount: (...a: unknown[]) => selectAccount(...a) })) @@ -54,6 +56,7 @@ beforeEach(() => { requireCapability.mockResolvedValue(null) aiStatus.mockReturnValue({ configured: true }) gatherCandidates.mockResolvedValue([{ account: '5410', label: 'Material', vatTreatment: 'standard_25', source: 'counterparty_template', confidence: 0.9 }]) + gatherUnderlag.mockResolvedValue('Kvitto: Biltema, totalt 499 SEK.') selectAccount.mockResolvedValue({ account: '5410', category: null, vatTreatment: 'standard_25', reverseCharge: false, confidence: 0.86, modelConfidence: 'high', agreement: 1, reasoning: 'r', @@ -101,9 +104,18 @@ describe('POST /api/agent/categorize', () => { expect(b.data.account).toBe('5410') expect(b.data.confidence).toBe(0.86) expect(b.data.candidates[0].account).toBe('5410') - // entity type + vat_registered threaded from the company rows; underlag + samples passed through + // A caller-supplied underlag is used verbatim (no server gather). + expect(gatherUnderlag).not.toHaveBeenCalled() expect(selectAccount).toHaveBeenCalledWith( expect.objectContaining({ entityType: 'aktiebolag', vatRegistered: true, underlag: 'Biltema AB 499 kr', samples: 3 }), ) }) + + it('gathers underlag server-side when the caller did not supply it', async () => { + await POST(createMockRequest('/x', { method: 'POST', body: body() })) + expect(gatherUnderlag).toHaveBeenCalled() + expect(selectAccount).toHaveBeenCalledWith( + expect.objectContaining({ underlag: 'Kvitto: Biltema, totalt 499 SEK.' }), + ) + }) }) diff --git a/app/api/agent/categorize/route.ts b/app/api/agent/categorize/route.ts index a3eab29d..f0fa8fea 100644 --- a/app/api/agent/categorize/route.ts +++ b/app/api/agent/categorize/route.ts @@ -8,6 +8,7 @@ import { requireCapability } from '@/lib/entitlements/has-capability' import { CAPABILITY } from '@/lib/entitlements/keys' import { getAiStatus } from '@/lib/ai' import { gatherCandidates } from '@/lib/agent/categorize/candidates' +import { gatherUnderlag } from '@/lib/agent/categorize/underlag' import { selectAccount } from '@/lib/agent/categorize/select-account' import { getErrorMessage as getUserErrorMessage } from '@/lib/errors/get-error-message' import type { EntityType, Transaction } from '@/types' @@ -80,7 +81,7 @@ export async function POST(request: Request): Promise { const { data: tx } = await supabase .from('transactions') - .select('id, merchant_name, description, original_description, amount, date, currency, category, is_business') + .select('id, merchant_name, description, original_description, amount, date, currency, category, is_business, document_id') .eq('id', parsed.data.transaction_id) .eq('company_id', companyId) .maybeSingle() @@ -92,6 +93,18 @@ export async function POST(request: Request): Promise { ]) try { + // Gather the matched receipt/invoice text when the caller didn't supply it: + // this is what lifts the cold-start case — the model reads the actual + // supplier + line items, not just the bank line. Best-effort; '' if none. + const underlag = + parsed.data.underlag ?? + (await gatherUnderlag( + supabase, + companyId, + (tx as Transaction).id, + (tx as { document_id?: string | null }).document_id, + )) + const candidates = await gatherCandidates(supabase, companyId, tx as Transaction) const selection = await selectAccount({ transaction: { @@ -101,7 +114,7 @@ export async function POST(request: Request): Promise { date: (tx as Transaction).date, currency: (tx as Transaction).currency, }, - underlag: parsed.data.underlag, + underlag, candidates, entityType: ((company?.entity_type as EntityType | undefined) ?? 'enskild_firma'), vatRegistered: settings?.vat_registered ?? false, diff --git a/lib/agent/categorize/__tests__/underlag.test.ts b/lib/agent/categorize/__tests__/underlag.test.ts new file mode 100644 index 00000000..35da8762 --- /dev/null +++ b/lib/agent/categorize/__tests__/underlag.test.ts @@ -0,0 +1,95 @@ +import { describe, it, expect } from 'vitest' +import type { SupabaseClient } from '@supabase/supabase-js' +import { gatherUnderlag } from '../underlag' + +function makeSupabase(opts: { + receipts?: unknown[] + inbox?: unknown[] + doc?: unknown | null + throwOn?: string +}): SupabaseClient { + return { + from(table: string) { + const rejects = opts.throwOn === table + const chain = { + select: () => chain, + eq: () => chain, + maybeSingle: async () => { + if (rejects) throw new Error('db down') + return { data: table === 'document_attachments' ? (opts.doc ?? null) : null } + }, + // The awaited list query resolves with rows — or rejects async, exactly + // how a real supabase query fails (never a synchronous throw from .from). + then: (resolve: (v: { data: unknown[] }) => unknown, reject: (e: unknown) => unknown) => { + if (rejects) return reject(new Error('db down')) + return resolve({ + data: + table === 'receipts' ? (opts.receipts ?? []) : table === 'invoice_inbox_items' ? (opts.inbox ?? []) : [], + }) + }, + } + return chain + }, + } as unknown as SupabaseClient +} + +describe('gatherUnderlag', () => { + it('renders a matched receipt with amount, VAT and flags', async () => { + const supabase = makeSupabase({ + receipts: [ + { + merchant_name: 'Biltema', + receipt_date: '2026-08-12', + total_amount: 499, + vat_amount: 99.8, + currency: 'SEK', + is_restaurant: false, + is_systembolaget: false, + }, + ], + }) + const out = await gatherUnderlag(supabase, 'c1', 't1') + expect(out).toContain('Kvitto: Biltema') + expect(out).toContain('2026-08-12') + expect(out).toContain('totalt 499 SEK') + expect(out).toContain('moms 99.8 SEK') + }) + + it('renders an inbox invoice with supplier + line items', async () => { + const supabase = makeSupabase({ + inbox: [ + { + extracted_data: { + supplier: { name: 'Vercel Inc' }, + invoice: { invoiceDate: '2026-08-01', currency: 'USD' }, + totals: { total: 20, vatAmount: 0 }, + lineItems: [{ description: 'Pro plan' }, { description: 'Bandwidth' }], + }, + }, + ], + }) + const out = await gatherUnderlag(supabase, 'c1', 't1') + expect(out).toContain('leverantör Vercel Inc') + expect(out).toContain('totalt 20 USD') + expect(out).toContain('Rader: "Pro plan"; "Bandwidth"') + }) + + it('renders the transaction attached document when a documentId is given', async () => { + const supabase = makeSupabase({ + doc: { extracted_data: { supplier: { name: 'Telia' }, invoice: { currency: 'SEK' }, totals: { total: 349 } } }, + }) + const out = await gatherUnderlag(supabase, 'c1', 't1', 'doc-9') + expect(out).toContain('Bifogat underlag: leverantör Telia') + expect(out).toContain('totalt 349 SEK') + }) + + it('returns empty string when there is no underlag', async () => { + const out = await gatherUnderlag(makeSupabase({}), 'c1', 't1') + expect(out).toBe('') + }) + + it('is best-effort: a failing query yields empty string, not a throw', async () => { + const out = await gatherUnderlag(makeSupabase({ throwOn: 'receipts' }), 'c1', 't1') + expect(out).toBe('') + }) +}) diff --git a/lib/agent/categorize/underlag.ts b/lib/agent/categorize/underlag.ts new file mode 100644 index 00000000..1cbb6056 --- /dev/null +++ b/lib/agent/categorize/underlag.ts @@ -0,0 +1,122 @@ +import type { SupabaseClient } from '@supabase/supabase-js' +import { roundOre } from '@/lib/money' + +/** + * Gather the underlag (receipt / invoice text) matched to a transaction and + * render it as compact text for the account selector. + * + * This is the highest-leverage input for the cold-start majority: without it + * the model sees only the bank line (merchant + amount); with it, it sees the + * supplier name, line items and VAT off the actual receipt. Same sources the + * transaction.categorization intent reads — receipts.matched_transaction_id, + * invoice_inbox_items.matched_transaction_id, and the transaction's own + * attached document — but produced as a bounded string, not a tool loop. + * + * Company-scoped and best-effort: any failing query is skipped, never fatal. + * Returns '' when there is no underlag (the selector then works off the bank + * line + candidates as before). + */ + +const MAX_UNDERLAG_CHARS = 2500 +const MAX_LINE_ITEMS = 8 + +function fmt(amount: number | null | undefined, currency: string | null | undefined): string | null { + if (amount === null || amount === undefined) return null + return `${roundOre(Number(amount))} ${currency ?? 'SEK'}` +} + +function lineItemDescriptions(ex: Record): string[] { + const items = ex.lineItems + if (!Array.isArray(items)) return [] + const out: string[] = [] + for (const it of items.slice(0, MAX_LINE_ITEMS)) { + const d = (it as { description?: unknown }).description + if (typeof d === 'string' && d.trim()) out.push(d.trim()) + } + return out +} + +export async function gatherUnderlag( + supabase: SupabaseClient, + companyId: string, + transactionId: string, + documentId?: string | null, +): Promise { + const [receiptsRes, inboxRes, docRes] = await Promise.all([ + supabase + .from('receipts') + .select('merchant_name, receipt_date, total_amount, vat_amount, currency, is_restaurant, is_systembolaget') + .eq('company_id', companyId) + .eq('matched_transaction_id', transactionId), + supabase + .from('invoice_inbox_items') + .select('extracted_data') + .eq('company_id', companyId) + .eq('matched_transaction_id', transactionId), + documentId + ? supabase + .from('document_attachments') + .select('extracted_data') + .eq('id', documentId) + .eq('company_id', companyId) + .maybeSingle() + : Promise.resolve({ data: null }), + ]).catch(() => [{ data: null }, { data: null }, { data: null }] as const) + + const lines: string[] = [] + + // Receipts (receipt-scan extracted fields). + for (const r of ((receiptsRes as { data: unknown }).data ?? []) as { + merchant_name: string | null + receipt_date: string | null + total_amount: number | null + vat_amount: number | null + currency: string | null + is_restaurant: boolean | null + is_systembolaget: boolean | null + }[]) { + const parts: string[] = [] + if (r.merchant_name) parts.push(r.merchant_name) + if (r.receipt_date) parts.push(r.receipt_date) + const total = fmt(r.total_amount, r.currency) + if (total) parts.push(`totalt ${total}`) + const vat = fmt(r.vat_amount, r.currency) + if (vat) parts.push(`moms ${vat}`) + if (r.is_restaurant) parts.push('restaurang/representation') + if (r.is_systembolaget) parts.push('Systembolaget') + if (parts.length) lines.push(`Kvitto: ${parts.join(', ')}.`) + } + + // Invoice inbox items (structured extraction of an invoice/receipt). + for (const it of ((inboxRes as { data: unknown }).data ?? []) as { + extracted_data: Record | null + }[]) { + const ex = it.extracted_data + if (!ex) continue + lines.push(renderExtraction(ex, 'Faktura/kvitto (inkorg)')) + } + + // The transaction's own attached document. + const doc = (docRes as { data: { extracted_data?: Record | null } | null }).data + if (doc?.extracted_data) lines.push(renderExtraction(doc.extracted_data, 'Bifogat underlag')) + + return lines.filter(Boolean).join('\n').slice(0, MAX_UNDERLAG_CHARS).trim() +} + +function renderExtraction(ex: Record, label: string): string { + const supplier = (ex.supplier as { name?: string | null } | undefined) ?? null + const invoice = (ex.invoice as { invoiceDate?: string | null; currency?: string | null } | undefined) ?? null + const totals = (ex.totals as { total?: number | null; vatAmount?: number | null } | undefined) ?? null + + const parts: string[] = [] + if (supplier?.name) parts.push(`leverantör ${supplier.name}`) + if (invoice?.invoiceDate) parts.push(invoice.invoiceDate) + const total = fmt(totals?.total, invoice?.currency) + if (total) parts.push(`totalt ${total}`) + const vat = fmt(totals?.vatAmount, invoice?.currency) + if (vat) parts.push(`moms ${vat}`) + + const items = lineItemDescriptions(ex) + const head = `${label}: ${parts.join(', ') || 'utläst underlag'}.` + return items.length ? `${head} Rader: ${items.map((d) => `"${d}"`).join('; ')}.` : head +}