Files
accounted/lib/agent/categorize/__tests__/select-account.test.ts
T
Jakob Wennberg d3409183c0 fix(categorize): make confidence honest — backing-driven, not the model's word (#1791)
A backtest against real bookings (scripts/backtest-categorize.ts, read-only)
showed the selector reporting 0.95 on pure category guesses, so "säker" was a
lie: high-confidence picks were only ~52% accurate.

Confidence is now driven by DETERMINISTIC BACKING — the confidence of a
candidate that independently points at the chosen account — not the model's
verbalized confidence (which the backtest showed is ~always "high"):
- a BACKED pick takes the candidate's confidence, reduced only when the model
  itself is unsure;
- an UNBACKED pick (a category guess no candidate agreed with) is capped at 0.7,
  below the säker band (0.8) — a guess is never "säker", however sure the model
  claims to be.

Re-running the backtest: säker (conf ≥0.8) accuracy 52% → 73%, and it now fires
only on template-backed picks. Still not auto-book-grade (want ~95%), so
auto-book stays off until isotonic calibration on real approvals — but the band
is now honest, which is what makes the whole UX trustworthy.

Also adds the read-only backtest harness so we can re-measure after any change.
37 categorize tests green; lint + guards clean.

Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
2026-08-21 16:43:20 +02:00

189 lines
7.8 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { describe, it, expect, vi, beforeEach } from 'vitest'
import {
getDefaultAccountForCategory,
getDefaultVatTreatmentForCategory,
} from '@/lib/bookkeeping/category-mapping'
import type { AccountCandidate, SelectAccountInput } from '../select-account'
const generateStructured = vi.fn()
vi.mock('@/lib/ai', () => ({ getAiService: () => ({ generateStructured }) }))
import { selectAccount } from '../select-account'
function pick(
choice: string,
opts: { confidence?: 'high' | 'medium' | 'low'; reverse_charge?: boolean; reasoning?: string } = {},
) {
return {
value: {
reasoning: opts.reasoning ?? 'resonemang',
choice,
confidence: opts.confidence ?? 'high',
reverse_charge: opts.reverse_charge ?? false,
},
model: 'qwen3.8',
usage: {},
}
}
const CAND: AccountCandidate = {
account: '5410',
label: 'Förbrukningsinventarier',
vatTreatment: 'standard_25',
source: 'counterparty_template',
confidence: 0.9,
}
function input(over: Partial<SelectAccountInput> = {}): SelectAccountInput {
return {
transaction: { merchantName: 'Biltema', description: 'Kortköp Biltema', amount: -499, currency: 'SEK' },
candidates: [CAND],
entityType: 'aktiebolag',
vatRegistered: true,
samples: 1,
...over,
}
}
beforeEach(() => {
vi.clearAllMocks()
generateStructured.mockResolvedValue(pick('cand:0'))
})
describe('selectAccount', () => {
it('resolves a chosen candidate to its account + VAT, flagged fromCandidate', async () => {
const res = await selectAccount(input())
expect(res.account).toBe('5410')
expect(res.fromCandidate).toBe(true)
expect(res.vatTreatment).toBe('standard_25')
expect(res.choice).toEqual({ kind: 'candidate', account: '5410' })
expect(res.confidence).toBeGreaterThan(0)
expect(res.model).toBe('qwen3.8')
})
it('resolves a chosen category to the deterministic default account + VAT (novel path)', async () => {
generateStructured.mockResolvedValue(pick('cat:expense_software'))
const res = await selectAccount(input({ candidates: [] }))
expect(res.category).toBe('expense_software')
expect(res.account).toBe(getDefaultAccountForCategory('expense_software', 'aktiebolag'))
expect(res.vatTreatment).toBe(getDefaultVatTreatmentForCategory('expense_software'))
expect(res.fromCandidate).toBe(false)
})
it('needs_review yields no account and zero confidence', async () => {
generateStructured.mockResolvedValue(pick('needs_review'))
const res = await selectAccount(input())
expect(res.account).toBeNull()
expect(res.category).toBeNull()
expect(res.confidence).toBe(0)
expect(res.choice).toEqual({ kind: 'needs_review' })
})
it('degrades an unknown/hallucinated choice to needs_review', async () => {
generateStructured.mockResolvedValue(pick('cand:99'))
const res = await selectAccount(input())
expect(res.choice).toEqual({ kind: 'needs_review' })
expect(res.account).toBeNull()
})
it('applies reverse charge only for a VAT-registered company', async () => {
generateStructured.mockResolvedValue(pick('cat:expense_professional_services', { reverse_charge: true }))
const yes = await selectAccount(input({ candidates: [], vatRegistered: true }))
expect(yes.reverseCharge).toBe(true)
expect(yes.vatTreatment).toBe('reverse_charge')
generateStructured.mockResolvedValue(pick('cat:expense_professional_services', { reverse_charge: true }))
const no = await selectAccount(input({ candidates: [], vatRegistered: false }))
expect(no.vatTreatment).not.toBe('reverse_charge')
})
it('never applies reverse charge to a needs_review outcome', async () => {
generateStructured.mockResolvedValue(pick('needs_review', { reverse_charge: true }))
const res = await selectAccount(input())
expect(res.reverseCharge).toBe(false)
})
describe('self-consistency', () => {
it('defaults to 3 samples and majority-votes the winner', async () => {
generateStructured
.mockResolvedValueOnce(pick('cand:0'))
.mockResolvedValueOnce(pick('cand:0'))
.mockResolvedValueOnce(pick('cat:expense_other'))
const res = await selectAccount(input({ samples: undefined }))
expect(generateStructured).toHaveBeenCalledTimes(3)
expect(res.choice).toEqual({ kind: 'candidate', account: '5410' })
expect(res.agreement).toBe(0.67)
})
it('single sample has full agreement', async () => {
const res = await selectAccount(input({ samples: 1 }))
expect(res.agreement).toBe(1)
expect(generateStructured).toHaveBeenCalledTimes(1)
})
it('lower agreement lowers an unbacked (category-guess) confidence', async () => {
generateStructured
.mockResolvedValueOnce(pick('cat:expense_office', { confidence: 'high' }))
.mockResolvedValueOnce(pick('cat:expense_office', { confidence: 'high' }))
.mockResolvedValueOnce(pick('cat:expense_travel', { confidence: 'high' }))
const split = await selectAccount(input({ candidates: [], samples: 3 }))
// No candidate backs the pick → unbacked: 2/3 agreement × 0.7 (high) ≈ 0.47.
expect(split.confidence).toBeCloseTo(0.47, 1)
expect(split.agreement).toBe(0.67)
})
it('an unbacked category guess never reaches the säker band, however sure the model is', async () => {
generateStructured.mockResolvedValue(pick('cat:expense_software', { confidence: 'high' }))
const res = await selectAccount(input({ candidates: [], samples: 1 }))
// high model conf, full agreement, but no deterministic backing → capped at 0.7 (< 0.8).
expect(res.confidence).toBeLessThan(0.8)
expect(res.confidence).toBeCloseTo(0.7, 2)
})
it('a backed pick takes the candidate confidence, only reduced when the model is unsure', async () => {
// model says "low", but the chosen candidate is a 0.9 counterparty template.
generateStructured.mockResolvedValue(pick('cand:0', { confidence: 'low' }))
const res = await selectAccount(input({ samples: 1 }))
// backing 0.9 × agreement 1 × low-factor 0.75 = 0.675.
expect(res.confidence).toBeCloseTo(0.68, 2)
})
it('a backed pick the model is sure about keeps the full candidate confidence', async () => {
generateStructured.mockResolvedValue(pick('cand:0', { confidence: 'high' }))
const res = await selectAccount(input({ samples: 1 }))
expect(res.confidence).toBe(0.9) // 0.9 × 1 × 1
})
})
describe('prompt + schema', () => {
it('builds a closed enum of candidate ids + category ids + needs_review, reasoning first', async () => {
await selectAccount(input())
const call = generateStructured.mock.calls[0][0]
const props = call.schema.jsonSchema.properties
expect(Object.keys(props)[0]).toBe('reasoning') // reason-before-choice
expect(props.choice.enum).toContain('cand:0')
expect(props.choice.enum).toContain('cat:expense_software')
expect(props.choice.enum).toContain('needs_review')
expect(call.system).toContain('BAS')
})
it('puts the candidate and the underlag into the prompt', async () => {
await selectAccount(input({ underlag: 'Leverantör: Biltema AB\nSumma: 499 kr' }))
const prompt = generateStructured.mock.calls[0][0].prompt as string
expect(prompt).toContain('KANDIDATKONTON')
expect(prompt).toContain('konto 5410')
expect(prompt).toContain('Biltema')
expect(prompt).toContain('Underlag')
expect(prompt).toContain('Summa: 499 kr')
})
it('omits the candidate block when there are none', async () => {
generateStructured.mockResolvedValue(pick('cat:expense_other'))
await selectAccount(input({ candidates: [] }))
const prompt = generateStructured.mock.calls[0][0].prompt as string
expect(prompt).not.toContain('KANDIDATKONTON')
expect(prompt).toContain('KATEGORIER')
})
})
})