feat(receipt-hunt): let a model settle the pairs the arithmetic cannot (#1498)

The weights in the matcher are chosen by hand, and two of them were
adjusted last week because they produced a wrong answer on real data. A
pair agreeing to within 1%, from a merchant the matcher recognised, still
scored 0.62 because a date had drifted. That says more about the
constants than about the receipt.

So the formula keeps what it is good at and hands over what it is not.
Above 0.80 it decides alone: an exact amount from a recognised merchant
needs no second opinion, and buying one for every pair would be latency
and cost for a verdict nobody doubts. Below 0.60 nothing is asked either,
because inviting a model to rescue a pair the evidence does not support
is how plausible wrong answers get made. Only the band between goes for
adjudication, which on a real ledger was five pairs against one the
formula had already settled.

The question is a yes or no with a reason, never a score. An earlier
design in this feature asked a model to rate its own certainty and it
anchored on round numbers, which is what the calibration literature
predicts. Judging concrete evidence and explaining the judgement is a
different task.

A verdict is checked, not trusted: a key nobody asked about is dropped, a
pair answered twice keeps its first answer, and a failed call accepts
nothing, leaving the run exactly where the arithmetic left it. The
formula's own score is stored unflattered next to the verdict, because
dressing it up would hide the uncertainty that sent the pair for a second
opinion in the first place, and agent_metadata records which instrument
decided.

Adjudication runs on a dry run too. A provkörning is meant to show what a
real run would propose, and skipping it would show a smaller, different
answer than the one that lands.

Measured on a real ledger: five uncertain pairs asked, two accepted with
reasons a human can check ("Samma leverantör och belopp stämmer inom
rimlig valutamarginal"), three rejected. Proposals went from one to three.

Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

lib/receipt-hunt/__tests__/adjudicate.test.ts

@@ -0,0 +1,122 @@
+/**
+ * The second opinion on pairs the arithmetic could not settle.
+ *
+ * These tests are about what the verdict may not do: accept a pair nobody
+ * asked about, answer twice, or turn a failed call into approvals. A rejected
+ * pair simply does not appear, which leaves the run where the formula left it.
+ */
+import { describe, it, expect, vi, beforeEach } from 'vitest'
+import type { UncertainPair } from '../adjudicate'
+
+const mockCreate = vi.fn()
+vi.mock('@anthropic-ai/bedrock-sdk', () => ({
+  default: class {
+    messages = { create: (...args: unknown[]) => mockCreate(...args) }
+  },
+}))
+
+import { adjudicate } from '../adjudicate'
+
+function toolReply(input: unknown) {
+  return { content: [{ type: 'tool_use', name: 'verdicts', id: 'tu', input }] }
+}
+
+function pair(key = 't1::d1'): UncertainPair {
+  return {
+    key,
+    purchase: {
+      description: 'VERCEL INC',
+      amount: 541.2,
+      currency: 'SEK',
+      date: '2026-07-21',
+    },
+    receipt: {
+      vendor: 'Vercel Inc.',
+      total: 54.85,
+      currency: 'USD',
+      sekTotal: 534.65,
+      date: '2026-07-14',
+      fileName: 'Receipt-2955-0452.pdf',
+    },
+    confidence: 0.62,
+    matchReasons: ['Belopp ±1%', 'Handlare matchar'],
+  }
+}
+
+beforeEach(() => vi.clearAllMocks())
+
+describe('adjudicate', () => {
+  it('returns the pairs it accepted, with the reason a human will read', async () => {
+    mockCreate.mockResolvedValue(
+      toolReply({
+        verdicts: [{ key: 't1::d1', accept: true, reason: 'Samma leverantör, beloppet stämmer efter växelkurs.' }],
+      }),
+    )
+    const out = await adjudicate([pair()])
+    expect(out).toHaveLength(1)
+    expect(out[0].reason).toContain('växelkurs')
+  })
+
+  it('drops a pair it rejected rather than proposing it anyway', async () => {
+    mockCreate.mockResolvedValue(
+      toolReply({ verdicts: [{ key: 't1::d1', accept: false, reason: 'Fakturan avser en annan månad.' }] }),
+    )
+    await expect(adjudicate([pair()])).resolves.toEqual([])
+  })
+
+  it('never accepts a pair nobody asked about', async () => {
+    // A key we did not send would attach a document to a purchase that was
+    // never weighed against it.
+    mockCreate.mockResolvedValue(
+      toolReply({ verdicts: [{ key: 'invented::pair', accept: true, reason: 'x' }] }),
+    )
+    await expect(adjudicate([pair()])).resolves.toEqual([])
+  })
+
+  it('takes the first answer when a pair is answered twice', async () => {
+    mockCreate.mockResolvedValue(
+      toolReply({
+        verdicts: [
+          { key: 't1::d1', accept: true, reason: 'ja' },
+          { key: 't1::d1', accept: false, reason: 'nej' },
+        ],
+      }),
+    )
+    const out = await adjudicate([pair()])
+    expect(out).toHaveLength(1)
+  })
+
+  it('accepts nothing when the call fails', async () => {
+    // The run is left exactly where the arithmetic left it.
+    mockCreate.mockRejectedValue(new Error('bedrock timeout'))
+    await expect(adjudicate([pair()])).resolves.toEqual([])
+  })
+
+  it('does not call the model when the formula settled everything', async () => {
+    await expect(adjudicate([])).resolves.toEqual([])
+    expect(mockCreate).not.toHaveBeenCalled()
+  })
+
+  it('asks about the whole batch in one call', async () => {
+    // The pairs are independent, but a run holds a handful and a call each
+    // would be latency for nothing.
+    mockCreate.mockResolvedValue(toolReply({ verdicts: [] }))
+    await adjudicate([pair('a::1'), pair('b::2'), pair('c::3')])
+    expect(mockCreate).toHaveBeenCalledTimes(1)
+  })
+
+  it('shows the model both sides of the amount, including the converted one', async () => {
+    mockCreate.mockResolvedValue(toolReply({ verdicts: [] }))
+    await adjudicate([pair()])
+    const sent = JSON.stringify(mockCreate.mock.calls[0][0].messages[0].content)
+    expect(sent).toContain('534.65')
+    expect(sent).toContain('541.2')
+  })
+
+  it('accepts a verdict list the model sent as a JSON string', async () => {
+    mockCreate.mockResolvedValue(
+      toolReply({ verdicts: JSON.stringify([{ key: 't1::d1', accept: true, reason: 'ja' }]) }),
+    )
+    await expect(adjudicate([pair()])).resolves.toHaveLength(1)
+  })
+})

lib/receipt-hunt/adjudicate.ts

@@ -0,0 +1,209 @@
+/**
+ * The pairs the arithmetic cannot settle.
+ *
+ * A weighted formula is the right instrument for the clear cases: it is free,
+ * instant, reproducible years later for an audit, and it cannot invent a
+ * merchant. It is a poor instrument for the middle. Its weights are chosen by
+ * hand, and a pair agreeing to within 1% from a recognised merchant can still
+ * land at 0.62 because a date drifted, which says more about the constants than
+ * about the receipt.
+ *
+ * So the formula keeps what it is good at and hands over what it is not. Only
+ * the uncertain band is sent here, which on a real ledger was eight pairs
+ * against five it had already settled.
+ *
+ * The question asked is deliberately a yes or no with a reason, never a score.
+ * An earlier design in this feature asked the model to rate its own certainty
+ * and it anchored on round numbers, which the calibration literature predicts:
+ * verbalised confidence is badly calibrated and barely separates a model's
+ * right answers from its wrong ones. Judging concrete evidence and explaining
+ * the judgement is a different task, and one it is good at.
+ *
+ * Nothing here books anything. An accepted pair becomes the same proposal a
+ * human approves, carrying the reason so the approval is checking an argument
+ * rather than trusting a verdict.
+ */
+import AnthropicBedrock from '@anthropic-ai/bedrock-sdk'
+import { z } from 'zod'
+import { createLogger } from '@/lib/logger'
+
+const log = createLogger('receipt-hunt-adjudicate')
+
+const MODEL =
+  process.env.RECEIPT_HUNT_MODEL_ID ||
+  process.env.BEDROCK_MODEL_ID ||
+  'eu.anthropic.claude-sonnet-5'
+
+export interface UncertainPair {
+  /** Stable handle for this pair, opaque to the model beyond matching it back. */
+  key: string
+  purchase: {
+    description: string
+    amount: number
+    currency: string
+    date: string
+  }
+  receipt: {
+    vendor: string | null
+    total: number | null
+    currency: string | null
+    /** The total in kronor when a rate was resolved, so both sides compare. */
+    sekTotal?: number | null
+    date: string | null
+    fileName: string | null
+  }
+  /** What the formula made of it, as context rather than as an instruction. */
+  confidence: number
+  matchReasons: string[]
+}
+
+export interface Verdict {
+  key: string
+  accept: boolean
+  reason: string
+}
+
+const VerdictSchema = z.object({
+  verdicts: z.preprocess(
+    (v) => {
+      if (typeof v !== 'string') return v
+      try {
+        return JSON.parse(v)
+      } catch {
+        return v
+      }
+    },
+    z
+      .array(
+        z.object({
+          key: z.string().min(1),
+          accept: z.coerce.boolean(),
+          reason: z.string().min(1).max(300),
+        }),
+      )
+      .default([]),
+  ),
+})
+
+const TOOL = {
+  type: 'object',
+  properties: {
+    verdicts: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          key: { type: 'string', description: 'The pair key exactly as given.' },
+          accept: {
+            type: 'boolean',
+            description: 'True only if this document is the underlag for this purchase.',
+          },
+          reason: { type: 'string', description: 'One short sentence, in Swedish.' },
+        },
+        required: ['key', 'accept', 'reason'],
+      },
+    },
+  },
+  required: ['verdicts'],
+}
+
+const SYSTEM = `Du avgör om en handling hör till ett visst köp.
+
+Du får par som en beräkning inte kunde avgöra själv. För varje par: är den här
+handlingen underlaget för det här köpet? Svara ja eller nej och säg varför.
+
+Det här gör paren svåra, och inget av det är i sig skäl att säga nej:
+
+- Datumen glider. Ett kortköp bokförs hos banken dagar efter att det gjordes,
+  utrikes gärna en vecka, och ett vidarebefordrat kvitto bär köpets datum medan
+  kontoutdraget bär bokföringsdagen.
+- Beloppen kan skilja någon procent när kvittot är i annan valuta. Banken drog
+  ett omräknat belopp till sin egen kurs; vi har räknat om till Riksbankens.
+  Ett par procents skillnad är växelkursen, inte olika belopp.
+- Bankens text är inte ett handlarnamn. Den är avhuggen och innehåller
+  betalvägar: "ANTHROPIC* CLAUDE SUB" och "Anthropic, PBC" är samma leverantör.
+
+Säg nej när något faktiskt talar emot: fel storleksordning på beloppet, en
+handling som avser en annan period, eller en handlare som inte rimligen är
+samma. Säg nej också när du helt enkelt inte kan avgöra det: en människa läser
+ditt skäl och ett vagt ja kostar mer än ett ärligt nej.
+
+Beräkningens poäng och skäl finns med som bakgrund. Den har redan vägt in
+belopp, handlare och datum, så håll dig inte till den: du ser saker den inte
+kan väga.
+
+reason: en kort mening på svenska om varför paret hör ihop eller inte.`
+
+function client(): AnthropicBedrock {
+  return new AnthropicBedrock({ awsRegion: process.env.AWS_REGION })
+}
+
+/**
+ * Settle the pairs the formula could not.
+ *
+ * One call for the batch: the pairs are independent, but a run holds a handful
+ * of them and a call each would be latency for nothing.
+ *
+ * Returns only the pairs it was given, and only accepted ones. A failed call
+ * accepts nothing, which leaves the run exactly where the arithmetic left it.
+ */
+export async function adjudicate(pairs: readonly UncertainPair[]): Promise<Verdict[]> {
+  if (pairs.length === 0) return []
+
+  const known = new Set(pairs.map((p) => p.key))
+  const payload = {
+    pairs: pairs.map((p) => ({
+      key: p.key,
+      kop: {
+        text: p.purchase.description,
+        belopp: p.purchase.amount,
+        valuta: p.purchase.currency,
+        datum: p.purchase.date,
+      },
+      handling: {
+        leverantor: p.receipt.vendor,
+        belopp: p.receipt.total,
+        valuta: p.receipt.currency,
+        belopp_i_kronor: p.receipt.sekTotal ?? null,
+        datum: p.receipt.date,
+        fil: p.receipt.fileName,
+      },
+      berakningen_sa: { poang: p.confidence, skal: p.matchReasons },
+    })),
+  }
+
+  try {
+    const response = await client().messages.create({
+      model: MODEL,
+      max_tokens: 4096,
+      system: SYSTEM,
+      tools: [{ name: 'verdicts', description: 'Return the result in this exact shape.', input_schema: TOOL as never }],
+      tool_choice: { type: 'tool', name: 'verdicts' },
+      messages: [{ role: 'user', content: JSON.stringify(payload, null, 1) }],
+    })
+
+    const block = response.content.find((c) => c.type === 'tool_use')
+    if (!block || block.type !== 'tool_use') throw new Error('model did not use the tool')
+    const parsed = VerdictSchema.parse(block.input)
+
+    const seen = new Set<string>()
+    const out: Verdict[] = []
+    for (const v of parsed.verdicts) {
+      // Only pairs we asked about, and each answered once: a key we never sent
+      // would attach a document to a purchase nobody weighed.
+      if (!known.has(v.key) || seen.has(v.key)) continue
+      seen.add(v.key)
+      if (!v.accept) continue
+      out.push({ key: v.key, accept: true, reason: v.reason })
+    }
+
+    log.info('adjudicated uncertain pairs', { asked: pairs.length, accepted: out.length })
+    return out
+  } catch (error) {
+    log.warn('adjudication failed, proposing none of the uncertain pairs', {
+      pairs: pairs.length,
+      error: error instanceof Error ? error.message : String(error),
+    })
+    return []
+  }
+}

lib/receipt-hunt/hunt.ts

@@ -18,10 +18,13 @@ import { getRiskLevel } from '@/lib/pending-operations/risk-tiers'
 import { getMailSearchService } from '@/lib/mail-search/service'
 import { ingestMailCandidate } from './ingest'
 import { normalizeForMatch } from '@/lib/documents/core-receipt-matcher'
+import { adjudicate } from './adjudicate'
 import { attachSekTotals } from './fx'
 import { extractMailDocuments } from './mail-intelligence'
 import {
+  CERTAIN_CONFIDENCE,
   MAX_PROPOSALS_PER_RUN,
+  UNCERTAIN_FLOOR,
   canHaveEmailReceipt,
   receiptIdentity,
   worthFetching,
@@ -430,7 +433,56 @@ export async function huntCompany(
   }
   if (pool.length === 0) return base
 
-  const proposals = selectProposals(transactions, pool, suppression, limit)
+  // Collect the whole band the formula can speak to, then split it: what it is
+  // sure of goes straight through, what it is not gets a second opinion.
+  const scored = selectProposals(transactions, pool, suppression, limit, UNCERTAIN_FLOOR)
+  const certain = scored.filter((p) => p.confidence >= CERTAIN_CONFIDENCE)
+  const uncertain = scored.filter((p) => p.confidence < CERTAIN_CONFIDENCE)
+
+  // Adjudicated on a dry run too. A provkörning is supposed to show exactly
+  // what a real run would propose, and skipping the second opinion would show
+  // a different, smaller answer than the one that lands.
+  const byIdForPairs = new Map(transactions.map((t) => [t.id, t]))
+  const verdicts = await adjudicate(
+        uncertain.map((p) => {
+          const tx = byIdForPairs.get(p.transaction_id) as HuntTransaction
+          return {
+            key: `${p.transaction_id}::${p.document_id}`,
+            purchase: {
+              description: tx.merchant_name || tx.description || '',
+              amount: Math.abs(tx.amount ?? 0),
+              currency: tx.currency ?? 'SEK',
+              date: tx.date ?? '',
+            },
+            receipt: {
+              vendor: p.merchant_name,
+              total: p.total_amount,
+              currency: p.currency,
+              sekTotal: p.sek_total ?? null,
+              date: p.receipt_date,
+              fileName: fileNames.get(p.document_id) ?? null,
+            },
+            confidence: p.confidence,
+            matchReasons: p.matchReasons,
+      }
+    }),
+  )
+
+  const accepted = new Map(verdicts.map((v) => [v.key, v.reason]))
+  const adjudicated = uncertain
+    .filter((p) => accepted.has(`${p.transaction_id}::${p.document_id}`))
+    .map((p) => ({
+      ...p,
+      // The verdict replaces the arithmetic's reasons, because it is what a
+      // human is being asked to check. The score stays as the formula computed
+      // it, unflattered: it is a real record of what the arithmetic made of
+      // the pair, and dressing it up would hide the very uncertainty that sent
+      // the pair for a second opinion.
+      matchReasons: [accepted.get(`${p.transaction_id}::${p.document_id}`) as string],
+      wasAdjudicated: true,
+    }))
+
+  const proposals = [...certain, ...adjudicated]
   if (proposals.length === 0) return base
   if (dryRun) return { ...base, proposed: proposals.length, proposals }
   if (!userId) return { ...base, skippedNoOwner: true }
@@ -469,6 +521,9 @@ export async function huntCompany(
         inbox_item_id: proposal.inbox_item_id,
         confidence: proposal.confidence,
         match_reasons: proposal.matchReasons,
+        // Which instrument decided: the arithmetic alone, or a second opinion
+        // on a pair the arithmetic could not settle.
+        decided_by: proposal.wasAdjudicated ? 'adjudicator' : 'matcher',
       },
     }
   })

lib/receipt-hunt/select.ts

@@ -29,6 +29,22 @@ import {
  */
 export const HUNT_MIN_CONFIDENCE = 0.7
 
+/**
+ * Above this the arithmetic decides alone.
+ *
+ * An exact amount from a merchant the matcher recognises needs no second
+ * opinion, and paying for one on every pair would be latency and cost for a
+ * verdict nobody doubts.
+ */
+export const CERTAIN_CONFIDENCE = 0.8
+
+/**
+ * Below this nothing is worth a second opinion either: the signals disagree,
+ * and asking a model to rescue a pair the evidence does not support is how
+ * plausible-sounding wrong answers get made.
+ */
+export const UNCERTAIN_FLOOR = 0.6
+
 /**
  * How far clear the winner must be before we propose it.
  *
@@ -75,6 +91,10 @@ export interface HuntProposal {
   receipt_date: string | null
   total_amount: number | null
   currency: string | null
+  /** The total in kronor, when a rate was resolved: both sides then compare. */
+  sek_total?: number | null
+  /** Settled by a second opinion rather than by the formula alone. */
+  wasAdjudicated?: boolean
   /**
    * Where the document came from, carried through from the inbox item so the
    * proposal can say "found in your mailbox" rather than implying a human
@@ -125,6 +145,11 @@ export function selectProposals(
   pool: readonly HuntPoolItem[],
   suppression: SuppressionSets,
   limit: number = MAX_PROPOSALS_PER_RUN,
+  /**
+   * Lowest confidence worth returning. The caller lowers it to collect the
+   * band it intends to adjudicate; every other guard still applies.
+   */
+  minConfidence: number = HUNT_MIN_CONFIDENCE,
 ): HuntProposal[] {
   if (pool.length === 0) return []
 
@@ -154,7 +179,7 @@ export function selectProposals(
     if (scored.length === 0) continue
 
     const [winner, runnerUp] = scored
-    if (winner.confidence < HUNT_MIN_CONFIDENCE) continue
+    if (winner.confidence < minConfidence) continue
     if (runnerUp && winner.confidence - runnerUp.confidence < AMBIGUITY_MARGIN) continue
 
     const documentId = winner.document_id as string
@@ -181,6 +206,7 @@ export function selectProposals(
       receipt_date: winner.receipt_date,
       total_amount: winner.total_amount,
       currency: winner.currency,
+      sek_total: poolById.get(winner.inbox_item_id)?.sek_total ?? null,
     })
   }