From 88140e57747dfffcb5ec7af4e2a147d8846c1fea Mon Sep 17 00:00:00 2001 From: Jakob Wennberg Date: Mon, 10 Aug 2026 18:12:14 +0200 Subject: [PATCH] feat(receipt-hunt): let a model settle the pairs the arithmetic cannot (#1498) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The weights in the matcher are chosen by hand, and two of them were adjusted last week because they produced a wrong answer on real data. A pair agreeing to within 1%, from a merchant the matcher recognised, still scored 0.62 because a date had drifted. That says more about the constants than about the receipt. So the formula keeps what it is good at and hands over what it is not. Above 0.80 it decides alone: an exact amount from a recognised merchant needs no second opinion, and buying one for every pair would be latency and cost for a verdict nobody doubts. Below 0.60 nothing is asked either, because inviting a model to rescue a pair the evidence does not support is how plausible wrong answers get made. Only the band between goes for adjudication, which on a real ledger was five pairs against one the formula had already settled. The question is a yes or no with a reason, never a score. An earlier design in this feature asked a model to rate its own certainty and it anchored on round numbers, which is what the calibration literature predicts. Judging concrete evidence and explaining the judgement is a different task. A verdict is checked, not trusted: a key nobody asked about is dropped, a pair answered twice keeps its first answer, and a failed call accepts nothing, leaving the run exactly where the arithmetic left it. The formula's own score is stored unflattered next to the verdict, because dressing it up would hide the uncertainty that sent the pair for a second opinion in the first place, and agent_metadata records which instrument decided. Adjudication runs on a dry run too. A provkörning is meant to show what a real run would propose, and skipping it would show a smaller, different answer than the one that lands. Measured on a real ledger: five uncertain pairs asked, two accepted with reasons a human can check ("Samma leverantör och belopp stämmer inom rimlig valutamarginal"), three rejected. Proposals went from one to three. Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) --- lib/receipt-hunt/__tests__/adjudicate.test.ts | 122 ++++++++++ lib/receipt-hunt/adjudicate.ts | 209 ++++++++++++++++++ lib/receipt-hunt/hunt.ts | 57 ++++- lib/receipt-hunt/select.ts | 28 ++- 4 files changed, 414 insertions(+), 2 deletions(-) create mode 100644 lib/receipt-hunt/__tests__/adjudicate.test.ts create mode 100644 lib/receipt-hunt/adjudicate.ts diff --git a/lib/receipt-hunt/__tests__/adjudicate.test.ts b/lib/receipt-hunt/__tests__/adjudicate.test.ts new file mode 100644 index 00000000..dd717941 --- /dev/null +++ b/lib/receipt-hunt/__tests__/adjudicate.test.ts @@ -0,0 +1,122 @@ +/** + * The second opinion on pairs the arithmetic could not settle. + * + * These tests are about what the verdict may not do: accept a pair nobody + * asked about, answer twice, or turn a failed call into approvals. A rejected + * pair simply does not appear, which leaves the run where the formula left it. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import type { UncertainPair } from '../adjudicate' + +const mockCreate = vi.fn() +vi.mock('@anthropic-ai/bedrock-sdk', () => ({ + default: class { + messages = { create: (...args: unknown[]) => mockCreate(...args) } + }, +})) + +import { adjudicate } from '../adjudicate' + +function toolReply(input: unknown) { + return { content: [{ type: 'tool_use', name: 'verdicts', id: 'tu', input }] } +} + +function pair(key = 't1::d1'): UncertainPair { + return { + key, + purchase: { + description: 'VERCEL INC', + amount: 541.2, + currency: 'SEK', + date: '2026-07-21', + }, + receipt: { + vendor: 'Vercel Inc.', + total: 54.85, + currency: 'USD', + sekTotal: 534.65, + date: '2026-07-14', + fileName: 'Receipt-2955-0452.pdf', + }, + confidence: 0.62, + matchReasons: ['Belopp ±1%', 'Handlare matchar'], + } +} + +beforeEach(() => vi.clearAllMocks()) + +describe('adjudicate', () => { + it('returns the pairs it accepted, with the reason a human will read', async () => { + mockCreate.mockResolvedValue( + toolReply({ + verdicts: [{ key: 't1::d1', accept: true, reason: 'Samma leverantör, beloppet stämmer efter växelkurs.' }], + }), + ) + const out = await adjudicate([pair()]) + expect(out).toHaveLength(1) + expect(out[0].reason).toContain('växelkurs') + }) + + it('drops a pair it rejected rather than proposing it anyway', async () => { + mockCreate.mockResolvedValue( + toolReply({ verdicts: [{ key: 't1::d1', accept: false, reason: 'Fakturan avser en annan månad.' }] }), + ) + await expect(adjudicate([pair()])).resolves.toEqual([]) + }) + + it('never accepts a pair nobody asked about', async () => { + // A key we did not send would attach a document to a purchase that was + // never weighed against it. + mockCreate.mockResolvedValue( + toolReply({ verdicts: [{ key: 'invented::pair', accept: true, reason: 'x' }] }), + ) + await expect(adjudicate([pair()])).resolves.toEqual([]) + }) + + it('takes the first answer when a pair is answered twice', async () => { + mockCreate.mockResolvedValue( + toolReply({ + verdicts: [ + { key: 't1::d1', accept: true, reason: 'ja' }, + { key: 't1::d1', accept: false, reason: 'nej' }, + ], + }), + ) + const out = await adjudicate([pair()]) + expect(out).toHaveLength(1) + }) + + it('accepts nothing when the call fails', async () => { + // The run is left exactly where the arithmetic left it. + mockCreate.mockRejectedValue(new Error('bedrock timeout')) + await expect(adjudicate([pair()])).resolves.toEqual([]) + }) + + it('does not call the model when the formula settled everything', async () => { + await expect(adjudicate([])).resolves.toEqual([]) + expect(mockCreate).not.toHaveBeenCalled() + }) + + it('asks about the whole batch in one call', async () => { + // The pairs are independent, but a run holds a handful and a call each + // would be latency for nothing. + mockCreate.mockResolvedValue(toolReply({ verdicts: [] })) + await adjudicate([pair('a::1'), pair('b::2'), pair('c::3')]) + expect(mockCreate).toHaveBeenCalledTimes(1) + }) + + it('shows the model both sides of the amount, including the converted one', async () => { + mockCreate.mockResolvedValue(toolReply({ verdicts: [] })) + await adjudicate([pair()]) + const sent = JSON.stringify(mockCreate.mock.calls[0][0].messages[0].content) + expect(sent).toContain('534.65') + expect(sent).toContain('541.2') + }) + + it('accepts a verdict list the model sent as a JSON string', async () => { + mockCreate.mockResolvedValue( + toolReply({ verdicts: JSON.stringify([{ key: 't1::d1', accept: true, reason: 'ja' }]) }), + ) + await expect(adjudicate([pair()])).resolves.toHaveLength(1) + }) +}) diff --git a/lib/receipt-hunt/adjudicate.ts b/lib/receipt-hunt/adjudicate.ts new file mode 100644 index 00000000..e8c354b0 --- /dev/null +++ b/lib/receipt-hunt/adjudicate.ts @@ -0,0 +1,209 @@ +/** + * The pairs the arithmetic cannot settle. + * + * A weighted formula is the right instrument for the clear cases: it is free, + * instant, reproducible years later for an audit, and it cannot invent a + * merchant. It is a poor instrument for the middle. Its weights are chosen by + * hand, and a pair agreeing to within 1% from a recognised merchant can still + * land at 0.62 because a date drifted, which says more about the constants than + * about the receipt. + * + * So the formula keeps what it is good at and hands over what it is not. Only + * the uncertain band is sent here, which on a real ledger was eight pairs + * against five it had already settled. + * + * The question asked is deliberately a yes or no with a reason, never a score. + * An earlier design in this feature asked the model to rate its own certainty + * and it anchored on round numbers, which the calibration literature predicts: + * verbalised confidence is badly calibrated and barely separates a model's + * right answers from its wrong ones. Judging concrete evidence and explaining + * the judgement is a different task, and one it is good at. + * + * Nothing here books anything. An accepted pair becomes the same proposal a + * human approves, carrying the reason so the approval is checking an argument + * rather than trusting a verdict. + */ +import AnthropicBedrock from '@anthropic-ai/bedrock-sdk' +import { z } from 'zod' +import { createLogger } from '@/lib/logger' + +const log = createLogger('receipt-hunt-adjudicate') + +const MODEL = + process.env.RECEIPT_HUNT_MODEL_ID || + process.env.BEDROCK_MODEL_ID || + 'eu.anthropic.claude-sonnet-5' + +export interface UncertainPair { + /** Stable handle for this pair, opaque to the model beyond matching it back. */ + key: string + purchase: { + description: string + amount: number + currency: string + date: string + } + receipt: { + vendor: string | null + total: number | null + currency: string | null + /** The total in kronor when a rate was resolved, so both sides compare. */ + sekTotal?: number | null + date: string | null + fileName: string | null + } + /** What the formula made of it, as context rather than as an instruction. */ + confidence: number + matchReasons: string[] +} + +export interface Verdict { + key: string + accept: boolean + reason: string +} + +const VerdictSchema = z.object({ + verdicts: z.preprocess( + (v) => { + if (typeof v !== 'string') return v + try { + return JSON.parse(v) + } catch { + return v + } + }, + z + .array( + z.object({ + key: z.string().min(1), + accept: z.coerce.boolean(), + reason: z.string().min(1).max(300), + }), + ) + .default([]), + ), +}) + +const TOOL = { + type: 'object', + properties: { + verdicts: { + type: 'array', + items: { + type: 'object', + properties: { + key: { type: 'string', description: 'The pair key exactly as given.' }, + accept: { + type: 'boolean', + description: 'True only if this document is the underlag for this purchase.', + }, + reason: { type: 'string', description: 'One short sentence, in Swedish.' }, + }, + required: ['key', 'accept', 'reason'], + }, + }, + }, + required: ['verdicts'], +} + +const SYSTEM = `Du avgör om en handling hör till ett visst köp. + +Du får par som en beräkning inte kunde avgöra själv. För varje par: är den här +handlingen underlaget för det här köpet? Svara ja eller nej och säg varför. + +Det här gör paren svåra, och inget av det är i sig skäl att säga nej: + +- Datumen glider. Ett kortköp bokförs hos banken dagar efter att det gjordes, + utrikes gärna en vecka, och ett vidarebefordrat kvitto bär köpets datum medan + kontoutdraget bär bokföringsdagen. +- Beloppen kan skilja någon procent när kvittot är i annan valuta. Banken drog + ett omräknat belopp till sin egen kurs; vi har räknat om till Riksbankens. + Ett par procents skillnad är växelkursen, inte olika belopp. +- Bankens text är inte ett handlarnamn. Den är avhuggen och innehåller + betalvägar: "ANTHROPIC* CLAUDE SUB" och "Anthropic, PBC" är samma leverantör. + +Säg nej när något faktiskt talar emot: fel storleksordning på beloppet, en +handling som avser en annan period, eller en handlare som inte rimligen är +samma. Säg nej också när du helt enkelt inte kan avgöra det: en människa läser +ditt skäl och ett vagt ja kostar mer än ett ärligt nej. + +Beräkningens poäng och skäl finns med som bakgrund. Den har redan vägt in +belopp, handlare och datum, så håll dig inte till den: du ser saker den inte +kan väga. + +reason: en kort mening på svenska om varför paret hör ihop eller inte.` + +function client(): AnthropicBedrock { + return new AnthropicBedrock({ awsRegion: process.env.AWS_REGION }) +} + +/** + * Settle the pairs the formula could not. + * + * One call for the batch: the pairs are independent, but a run holds a handful + * of them and a call each would be latency for nothing. + * + * Returns only the pairs it was given, and only accepted ones. A failed call + * accepts nothing, which leaves the run exactly where the arithmetic left it. + */ +export async function adjudicate(pairs: readonly UncertainPair[]): Promise { + if (pairs.length === 0) return [] + + const known = new Set(pairs.map((p) => p.key)) + const payload = { + pairs: pairs.map((p) => ({ + key: p.key, + kop: { + text: p.purchase.description, + belopp: p.purchase.amount, + valuta: p.purchase.currency, + datum: p.purchase.date, + }, + handling: { + leverantor: p.receipt.vendor, + belopp: p.receipt.total, + valuta: p.receipt.currency, + belopp_i_kronor: p.receipt.sekTotal ?? null, + datum: p.receipt.date, + fil: p.receipt.fileName, + }, + berakningen_sa: { poang: p.confidence, skal: p.matchReasons }, + })), + } + + try { + const response = await client().messages.create({ + model: MODEL, + max_tokens: 4096, + system: SYSTEM, + tools: [{ name: 'verdicts', description: 'Return the result in this exact shape.', input_schema: TOOL as never }], + tool_choice: { type: 'tool', name: 'verdicts' }, + messages: [{ role: 'user', content: JSON.stringify(payload, null, 1) }], + }) + + const block = response.content.find((c) => c.type === 'tool_use') + if (!block || block.type !== 'tool_use') throw new Error('model did not use the tool') + const parsed = VerdictSchema.parse(block.input) + + const seen = new Set() + const out: Verdict[] = [] + for (const v of parsed.verdicts) { + // Only pairs we asked about, and each answered once: a key we never sent + // would attach a document to a purchase nobody weighed. + if (!known.has(v.key) || seen.has(v.key)) continue + seen.add(v.key) + if (!v.accept) continue + out.push({ key: v.key, accept: true, reason: v.reason }) + } + + log.info('adjudicated uncertain pairs', { asked: pairs.length, accepted: out.length }) + return out + } catch (error) { + log.warn('adjudication failed, proposing none of the uncertain pairs', { + pairs: pairs.length, + error: error instanceof Error ? error.message : String(error), + }) + return [] + } +} diff --git a/lib/receipt-hunt/hunt.ts b/lib/receipt-hunt/hunt.ts index 66ef967c..418b30a0 100644 --- a/lib/receipt-hunt/hunt.ts +++ b/lib/receipt-hunt/hunt.ts @@ -18,10 +18,13 @@ import { getRiskLevel } from '@/lib/pending-operations/risk-tiers' import { getMailSearchService } from '@/lib/mail-search/service' import { ingestMailCandidate } from './ingest' import { normalizeForMatch } from '@/lib/documents/core-receipt-matcher' +import { adjudicate } from './adjudicate' import { attachSekTotals } from './fx' import { extractMailDocuments } from './mail-intelligence' import { + CERTAIN_CONFIDENCE, MAX_PROPOSALS_PER_RUN, + UNCERTAIN_FLOOR, canHaveEmailReceipt, receiptIdentity, worthFetching, @@ -430,7 +433,56 @@ export async function huntCompany( } if (pool.length === 0) return base - const proposals = selectProposals(transactions, pool, suppression, limit) + // Collect the whole band the formula can speak to, then split it: what it is + // sure of goes straight through, what it is not gets a second opinion. + const scored = selectProposals(transactions, pool, suppression, limit, UNCERTAIN_FLOOR) + const certain = scored.filter((p) => p.confidence >= CERTAIN_CONFIDENCE) + const uncertain = scored.filter((p) => p.confidence < CERTAIN_CONFIDENCE) + + // Adjudicated on a dry run too. A provkörning is supposed to show exactly + // what a real run would propose, and skipping the second opinion would show + // a different, smaller answer than the one that lands. + const byIdForPairs = new Map(transactions.map((t) => [t.id, t])) + const verdicts = await adjudicate( + uncertain.map((p) => { + const tx = byIdForPairs.get(p.transaction_id) as HuntTransaction + return { + key: `${p.transaction_id}::${p.document_id}`, + purchase: { + description: tx.merchant_name || tx.description || '', + amount: Math.abs(tx.amount ?? 0), + currency: tx.currency ?? 'SEK', + date: tx.date ?? '', + }, + receipt: { + vendor: p.merchant_name, + total: p.total_amount, + currency: p.currency, + sekTotal: p.sek_total ?? null, + date: p.receipt_date, + fileName: fileNames.get(p.document_id) ?? null, + }, + confidence: p.confidence, + matchReasons: p.matchReasons, + } + }), + ) + + const accepted = new Map(verdicts.map((v) => [v.key, v.reason])) + const adjudicated = uncertain + .filter((p) => accepted.has(`${p.transaction_id}::${p.document_id}`)) + .map((p) => ({ + ...p, + // The verdict replaces the arithmetic's reasons, because it is what a + // human is being asked to check. The score stays as the formula computed + // it, unflattered: it is a real record of what the arithmetic made of + // the pair, and dressing it up would hide the very uncertainty that sent + // the pair for a second opinion. + matchReasons: [accepted.get(`${p.transaction_id}::${p.document_id}`) as string], + wasAdjudicated: true, + })) + + const proposals = [...certain, ...adjudicated] if (proposals.length === 0) return base if (dryRun) return { ...base, proposed: proposals.length, proposals } if (!userId) return { ...base, skippedNoOwner: true } @@ -469,6 +521,9 @@ export async function huntCompany( inbox_item_id: proposal.inbox_item_id, confidence: proposal.confidence, match_reasons: proposal.matchReasons, + // Which instrument decided: the arithmetic alone, or a second opinion + // on a pair the arithmetic could not settle. + decided_by: proposal.wasAdjudicated ? 'adjudicator' : 'matcher', }, } }) diff --git a/lib/receipt-hunt/select.ts b/lib/receipt-hunt/select.ts index 92a514c8..fc04cb6d 100644 --- a/lib/receipt-hunt/select.ts +++ b/lib/receipt-hunt/select.ts @@ -29,6 +29,22 @@ import { */ export const HUNT_MIN_CONFIDENCE = 0.7 +/** + * Above this the arithmetic decides alone. + * + * An exact amount from a merchant the matcher recognises needs no second + * opinion, and paying for one on every pair would be latency and cost for a + * verdict nobody doubts. + */ +export const CERTAIN_CONFIDENCE = 0.8 + +/** + * Below this nothing is worth a second opinion either: the signals disagree, + * and asking a model to rescue a pair the evidence does not support is how + * plausible-sounding wrong answers get made. + */ +export const UNCERTAIN_FLOOR = 0.6 + /** * How far clear the winner must be before we propose it. * @@ -75,6 +91,10 @@ export interface HuntProposal { receipt_date: string | null total_amount: number | null currency: string | null + /** The total in kronor, when a rate was resolved: both sides then compare. */ + sek_total?: number | null + /** Settled by a second opinion rather than by the formula alone. */ + wasAdjudicated?: boolean /** * Where the document came from, carried through from the inbox item so the * proposal can say "found in your mailbox" rather than implying a human @@ -125,6 +145,11 @@ export function selectProposals( pool: readonly HuntPoolItem[], suppression: SuppressionSets, limit: number = MAX_PROPOSALS_PER_RUN, + /** + * Lowest confidence worth returning. The caller lowers it to collect the + * band it intends to adjudicate; every other guard still applies. + */ + minConfidence: number = HUNT_MIN_CONFIDENCE, ): HuntProposal[] { if (pool.length === 0) return [] @@ -154,7 +179,7 @@ export function selectProposals( if (scored.length === 0) continue const [winner, runnerUp] = scored - if (winner.confidence < HUNT_MIN_CONFIDENCE) continue + if (winner.confidence < minConfidence) continue if (runnerUp && winner.confidence - runnerUp.confidence < AMBIGUITY_MARGIN) continue const documentId = winner.document_id as string @@ -181,6 +206,7 @@ export function selectProposals( receipt_date: winner.receipt_date, total_amount: winner.total_amount, currency: winner.currency, + sek_total: poolById.get(winner.inbox_item_id)?.sek_total ?? null, }) }