fix(categorize): make confidence honest — backing-driven, not the model's word (#1791)

A backtest against real bookings (scripts/backtest-categorize.ts, read-only)
showed the selector reporting 0.95 on pure category guesses, so "säker" was a
lie: high-confidence picks were only ~52% accurate.

Confidence is now driven by DETERMINISTIC BACKING — the confidence of a
candidate that independently points at the chosen account — not the model's
verbalized confidence (which the backtest showed is ~always "high"):
- a BACKED pick takes the candidate's confidence, reduced only when the model
  itself is unsure;
- an UNBACKED pick (a category guess no candidate agreed with) is capped at 0.7,
  below the säker band (0.8) — a guess is never "säker", however sure the model
  claims to be.

Re-running the backtest: säker (conf ≥0.8) accuracy 52% → 73%, and it now fires
only on template-backed picks. Still not auto-book-grade (want ~95%), so
auto-book stays off until isotonic calibration on real approvals — but the band
is now honest, which is what makes the whole UX trustworthy.

Also adds the read-only backtest harness so we can re-measure after any change.
37 categorize tests green; lint + guards clean.

Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

lib/agent/categorize/__tests__/select-account.test.ts

@@ -121,23 +121,37 @@ describe('selectAccount', () => {
       expect(generateStructured).toHaveBeenCalledTimes(1)
     })
 
-    it('lower agreement lowers the combined confidence', async () => {
+    it('lower agreement lowers an unbacked (category-guess) confidence', async () => {
       generateStructured
         .mockResolvedValueOnce(pick('cat:expense_office', { confidence: 'high' }))
         .mockResolvedValueOnce(pick('cat:expense_office', { confidence: 'high' }))
         .mockResolvedValueOnce(pick('cat:expense_travel', { confidence: 'high' }))
       const split = await selectAccount(input({ candidates: [], samples: 3 }))
-      // 2/3 agreement × 0.95 model weight
-      expect(split.confidence).toBeCloseTo(0.63, 1)
+      // No candidate backs the pick → unbacked: 2/3 agreement × 0.7 (high) ≈ 0.47.
+      expect(split.confidence).toBeCloseTo(0.47, 1)
       expect(split.agreement).toBe(0.67)
     })
 
-    it('a high-confidence candidate floors the score at its deterministic confidence × agreement', async () => {
-      // model says "low", but the chosen candidate is a 0.9 counterparty template
+    it('an unbacked category guess never reaches the säker band, however sure the model is', async () => {
+      generateStructured.mockResolvedValue(pick('cat:expense_software', { confidence: 'high' }))
+      const res = await selectAccount(input({ candidates: [], samples: 1 }))
+      // high model conf, full agreement, but no deterministic backing → capped at 0.7 (< 0.8).
+      expect(res.confidence).toBeLessThan(0.8)
+      expect(res.confidence).toBeCloseTo(0.7, 2)
+    })
+
+    it('a backed pick takes the candidate confidence, only reduced when the model is unsure', async () => {
+      // model says "low", but the chosen candidate is a 0.9 counterparty template.
       generateStructured.mockResolvedValue(pick('cand:0', { confidence: 'low' }))
       const res = await selectAccount(input({ samples: 1 }))
-      // low weight 0.5 vs candidate 0.9 × agreement 1 → floored to 0.9
-      expect(res.confidence).toBe(0.9)
+      // backing 0.9 × agreement 1 × low-factor 0.75 = 0.675.
+      expect(res.confidence).toBeCloseTo(0.68, 2)
+    })
+
+    it('a backed pick the model is sure about keeps the full candidate confidence', async () => {
+      generateStructured.mockResolvedValue(pick('cand:0', { confidence: 'high' }))
+      const res = await selectAccount(input({ samples: 1 }))
+      expect(res.confidence).toBe(0.9) // 0.9 × 1 × 1
     })
   })
 

lib/agent/categorize/select-account.ts

@@ -256,10 +256,22 @@ function parsePick(value: unknown, validIds: Set<string>): RawPick {
   }
 }
 
-const MODEL_CONF_WEIGHT: Record<'high' | 'medium' | 'low', number> = {
-  high: 0.95,
-  medium: 0.75,
-  low: 0.5,
+// A backtest against real bookings showed the model reports "high" almost
+// always, so its verbalized confidence can't drive the score. Instead:
+// - a pick BACKED by a deterministic candidate takes that candidate's
+//   confidence (the retrieval signal), only pulled down when the model itself
+//   is unsure;
+// - an UNBACKED pick (a pure category guess no candidate agreed with) is capped
+//   below the "säker" threshold, however sure the model claims to be.
+const BACKED_MODEL_FACTOR: Record<'high' | 'medium' | 'low', number> = {
+  high: 1,
+  medium: 0.9,
+  low: 0.75,
+}
+const UNBACKED_CONFIDENCE: Record<'high' | 'medium' | 'low', number> = {
+  high: 0.7, // stays under the säker band (0.8): a guess is never "säker"
+  medium: 0.5,
+  low: 0.3,
 }
 
 /**
@@ -313,13 +325,11 @@ export async function selectAccount(input: SelectAccountInput): Promise<AccountS
   let category: TransactionCategory | null = null
   let vatTreatment: VatTreatment | null = null
   let fromCandidate = false
-  let candidateConfidence = 0
 
   if (choice.kind === 'candidate') {
     const cand = input.candidates.find((c) => c.account === choice.account)
     account = choice.account
     fromCandidate = true
-    candidateConfidence = cand?.confidence ?? 0
     vatTreatment = cand?.vatTreatment ?? null
   } else if (choice.kind === 'category') {
     category = choice.category
@@ -332,12 +342,25 @@ export async function selectAccount(input: SelectAccountInput): Promise<AccountS
   const reverseCharge = winningSample.reverseCharge && choice.kind !== 'needs_review'
   if (reverseCharge && input.vatRegistered) vatTreatment = 'reverse_charge'
 
-  // Combined confidence: model conf × agreement, floored by the deterministic
-  // candidate signal when a known candidate won. needs_review is always 0.
+  // Confidence is driven by DETERMINISTIC BACKING, not the model's self-report.
+  // "backing" = the confidence of a candidate that independently points at the
+  // chosen account (a candidate pick backs itself; a category pick is backed
+  // only if a candidate resolved to the same account). needs_review is 0.
+  const backing =
+    choice.kind === 'needs_review'
+      ? 0
+      : (input.candidates.find((c) => c.account === account)?.confidence ?? 0)
+
   let confidence = 0
   if (choice.kind !== 'needs_review') {
-    confidence = MODEL_CONF_WEIGHT[winningSample.confidence] * agreement
-    if (fromCandidate) confidence = Math.max(confidence, candidateConfidence * agreement)
+    confidence =
+      backing > 0
+        ? // Memory backs the pick → the retrieval signal, tempered by self-
+          // consistency; the model's own confidence only pulls it down when low.
+          agreement * backing * BACKED_MODEL_FACTOR[winningSample.confidence]
+        : // Pure model guess → capped below "säker"; verbalized confidence is
+          // not trusted to lift an unbacked pick past a suggestion.
+          agreement * UNBACKED_CONFIDENCE[winningSample.confidence]
     confidence = roundOre(Math.min(1, confidence))
   }