diff --git a/lib/receipt-hunt/__tests__/adjudicate.test.ts b/lib/receipt-hunt/__tests__/adjudicate.test.ts new file mode 100644 index 00000000..dd717941 --- /dev/null +++ b/lib/receipt-hunt/__tests__/adjudicate.test.ts @@ -0,0 +1,122 @@ +/** + * The second opinion on pairs the arithmetic could not settle. + * + * These tests are about what the verdict may not do: accept a pair nobody + * asked about, answer twice, or turn a failed call into approvals. A rejected + * pair simply does not appear, which leaves the run where the formula left it. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import type { UncertainPair } from '../adjudicate' + +const mockCreate = vi.fn() +vi.mock('@anthropic-ai/bedrock-sdk', () => ({ + default: class { + messages = { create: (...args: unknown[]) => mockCreate(...args) } + }, +})) + +import { adjudicate } from '../adjudicate' + +function toolReply(input: unknown) { + return { content: [{ type: 'tool_use', name: 'verdicts', id: 'tu', input }] } +} + +function pair(key = 't1::d1'): UncertainPair { + return { + key, + purchase: { + description: 'VERCEL INC', + amount: 541.2, + currency: 'SEK', + date: '2026-07-21', + }, + receipt: { + vendor: 'Vercel Inc.', + total: 54.85, + currency: 'USD', + sekTotal: 534.65, + date: '2026-07-14', + fileName: 'Receipt-2955-0452.pdf', + }, + confidence: 0.62, + matchReasons: ['Belopp ±1%', 'Handlare matchar'], + } +} + +beforeEach(() => vi.clearAllMocks()) + +describe('adjudicate', () => { + it('returns the pairs it accepted, with the reason a human will read', async () => { + mockCreate.mockResolvedValue( + toolReply({ + verdicts: [{ key: 't1::d1', accept: true, reason: 'Samma leverantör, beloppet stämmer efter växelkurs.' }], + }), + ) + const out = await adjudicate([pair()]) + expect(out).toHaveLength(1) + expect(out[0].reason).toContain('växelkurs') + }) + + it('drops a pair it rejected rather than proposing it anyway', async () => { + mockCreate.mockResolvedValue( + toolReply({ verdicts: [{ key: 't1::d1', accept: false, reason: 'Fakturan avser en annan månad.' }] }), + ) + await expect(adjudicate([pair()])).resolves.toEqual([]) + }) + + it('never accepts a pair nobody asked about', async () => { + // A key we did not send would attach a document to a purchase that was + // never weighed against it. + mockCreate.mockResolvedValue( + toolReply({ verdicts: [{ key: 'invented::pair', accept: true, reason: 'x' }] }), + ) + await expect(adjudicate([pair()])).resolves.toEqual([]) + }) + + it('takes the first answer when a pair is answered twice', async () => { + mockCreate.mockResolvedValue( + toolReply({ + verdicts: [ + { key: 't1::d1', accept: true, reason: 'ja' }, + { key: 't1::d1', accept: false, reason: 'nej' }, + ], + }), + ) + const out = await adjudicate([pair()]) + expect(out).toHaveLength(1) + }) + + it('accepts nothing when the call fails', async () => { + // The run is left exactly where the arithmetic left it. + mockCreate.mockRejectedValue(new Error('bedrock timeout')) + await expect(adjudicate([pair()])).resolves.toEqual([]) + }) + + it('does not call the model when the formula settled everything', async () => { + await expect(adjudicate([])).resolves.toEqual([]) + expect(mockCreate).not.toHaveBeenCalled() + }) + + it('asks about the whole batch in one call', async () => { + // The pairs are independent, but a run holds a handful and a call each + // would be latency for nothing. + mockCreate.mockResolvedValue(toolReply({ verdicts: [] })) + await adjudicate([pair('a::1'), pair('b::2'), pair('c::3')]) + expect(mockCreate).toHaveBeenCalledTimes(1) + }) + + it('shows the model both sides of the amount, including the converted one', async () => { + mockCreate.mockResolvedValue(toolReply({ verdicts: [] })) + await adjudicate([pair()]) + const sent = JSON.stringify(mockCreate.mock.calls[0][0].messages[0].content) + expect(sent).toContain('534.65') + expect(sent).toContain('541.2') + }) + + it('accepts a verdict list the model sent as a JSON string', async () => { + mockCreate.mockResolvedValue( + toolReply({ verdicts: JSON.stringify([{ key: 't1::d1', accept: true, reason: 'ja' }]) }), + ) + await expect(adjudicate([pair()])).resolves.toHaveLength(1) + }) +}) diff --git a/lib/receipt-hunt/adjudicate.ts b/lib/receipt-hunt/adjudicate.ts new file mode 100644 index 00000000..e8c354b0 --- /dev/null +++ b/lib/receipt-hunt/adjudicate.ts @@ -0,0 +1,209 @@ +/** + * The pairs the arithmetic cannot settle. + * + * A weighted formula is the right instrument for the clear cases: it is free, + * instant, reproducible years later for an audit, and it cannot invent a + * merchant. It is a poor instrument for the middle. Its weights are chosen by + * hand, and a pair agreeing to within 1% from a recognised merchant can still + * land at 0.62 because a date drifted, which says more about the constants than + * about the receipt. + * + * So the formula keeps what it is good at and hands over what it is not. Only + * the uncertain band is sent here, which on a real ledger was eight pairs + * against five it had already settled. + * + * The question asked is deliberately a yes or no with a reason, never a score. + * An earlier design in this feature asked the model to rate its own certainty + * and it anchored on round numbers, which the calibration literature predicts: + * verbalised confidence is badly calibrated and barely separates a model's + * right answers from its wrong ones. Judging concrete evidence and explaining + * the judgement is a different task, and one it is good at. + * + * Nothing here books anything. An accepted pair becomes the same proposal a + * human approves, carrying the reason so the approval is checking an argument + * rather than trusting a verdict. + */ +import AnthropicBedrock from '@anthropic-ai/bedrock-sdk' +import { z } from 'zod' +import { createLogger } from '@/lib/logger' + +const log = createLogger('receipt-hunt-adjudicate') + +const MODEL = + process.env.RECEIPT_HUNT_MODEL_ID || + process.env.BEDROCK_MODEL_ID || + 'eu.anthropic.claude-sonnet-5' + +export interface UncertainPair { + /** Stable handle for this pair, opaque to the model beyond matching it back. */ + key: string + purchase: { + description: string + amount: number + currency: string + date: string + } + receipt: { + vendor: string | null + total: number | null + currency: string | null + /** The total in kronor when a rate was resolved, so both sides compare. */ + sekTotal?: number | null + date: string | null + fileName: string | null + } + /** What the formula made of it, as context rather than as an instruction. */ + confidence: number + matchReasons: string[] +} + +export interface Verdict { + key: string + accept: boolean + reason: string +} + +const VerdictSchema = z.object({ + verdicts: z.preprocess( + (v) => { + if (typeof v !== 'string') return v + try { + return JSON.parse(v) + } catch { + return v + } + }, + z + .array( + z.object({ + key: z.string().min(1), + accept: z.coerce.boolean(), + reason: z.string().min(1).max(300), + }), + ) + .default([]), + ), +}) + +const TOOL = { + type: 'object', + properties: { + verdicts: { + type: 'array', + items: { + type: 'object', + properties: { + key: { type: 'string', description: 'The pair key exactly as given.' }, + accept: { + type: 'boolean', + description: 'True only if this document is the underlag for this purchase.', + }, + reason: { type: 'string', description: 'One short sentence, in Swedish.' }, + }, + required: ['key', 'accept', 'reason'], + }, + }, + }, + required: ['verdicts'], +} + +const SYSTEM = `Du avgör om en handling hör till ett visst köp. + +Du får par som en beräkning inte kunde avgöra själv. För varje par: är den här +handlingen underlaget för det här köpet? Svara ja eller nej och säg varför. + +Det här gör paren svåra, och inget av det är i sig skäl att säga nej: + +- Datumen glider. Ett kortköp bokförs hos banken dagar efter att det gjordes, + utrikes gärna en vecka, och ett vidarebefordrat kvitto bär köpets datum medan + kontoutdraget bär bokföringsdagen. +- Beloppen kan skilja någon procent när kvittot är i annan valuta. Banken drog + ett omräknat belopp till sin egen kurs; vi har räknat om till Riksbankens. + Ett par procents skillnad är växelkursen, inte olika belopp. +- Bankens text är inte ett handlarnamn. Den är avhuggen och innehåller + betalvägar: "ANTHROPIC* CLAUDE SUB" och "Anthropic, PBC" är samma leverantör. + +Säg nej när något faktiskt talar emot: fel storleksordning på beloppet, en +handling som avser en annan period, eller en handlare som inte rimligen är +samma. Säg nej också när du helt enkelt inte kan avgöra det: en människa läser +ditt skäl och ett vagt ja kostar mer än ett ärligt nej. + +Beräkningens poäng och skäl finns med som bakgrund. Den har redan vägt in +belopp, handlare och datum, så håll dig inte till den: du ser saker den inte +kan väga. + +reason: en kort mening på svenska om varför paret hör ihop eller inte.` + +function client(): AnthropicBedrock { + return new AnthropicBedrock({ awsRegion: process.env.AWS_REGION }) +} + +/** + * Settle the pairs the formula could not. + * + * One call for the batch: the pairs are independent, but a run holds a handful + * of them and a call each would be latency for nothing. + * + * Returns only the pairs it was given, and only accepted ones. A failed call + * accepts nothing, which leaves the run exactly where the arithmetic left it. + */ +export async function adjudicate(pairs: readonly UncertainPair[]): Promise { + if (pairs.length === 0) return [] + + const known = new Set(pairs.map((p) => p.key)) + const payload = { + pairs: pairs.map((p) => ({ + key: p.key, + kop: { + text: p.purchase.description, + belopp: p.purchase.amount, + valuta: p.purchase.currency, + datum: p.purchase.date, + }, + handling: { + leverantor: p.receipt.vendor, + belopp: p.receipt.total, + valuta: p.receipt.currency, + belopp_i_kronor: p.receipt.sekTotal ?? null, + datum: p.receipt.date, + fil: p.receipt.fileName, + }, + berakningen_sa: { poang: p.confidence, skal: p.matchReasons }, + })), + } + + try { + const response = await client().messages.create({ + model: MODEL, + max_tokens: 4096, + system: SYSTEM, + tools: [{ name: 'verdicts', description: 'Return the result in this exact shape.', input_schema: TOOL as never }], + tool_choice: { type: 'tool', name: 'verdicts' }, + messages: [{ role: 'user', content: JSON.stringify(payload, null, 1) }], + }) + + const block = response.content.find((c) => c.type === 'tool_use') + if (!block || block.type !== 'tool_use') throw new Error('model did not use the tool') + const parsed = VerdictSchema.parse(block.input) + + const seen = new Set() + const out: Verdict[] = [] + for (const v of parsed.verdicts) { + // Only pairs we asked about, and each answered once: a key we never sent + // would attach a document to a purchase nobody weighed. + if (!known.has(v.key) || seen.has(v.key)) continue + seen.add(v.key) + if (!v.accept) continue + out.push({ key: v.key, accept: true, reason: v.reason }) + } + + log.info('adjudicated uncertain pairs', { asked: pairs.length, accepted: out.length }) + return out + } catch (error) { + log.warn('adjudication failed, proposing none of the uncertain pairs', { + pairs: pairs.length, + error: error instanceof Error ? error.message : String(error), + }) + return [] + } +} diff --git a/lib/receipt-hunt/hunt.ts b/lib/receipt-hunt/hunt.ts index 66ef967c..418b30a0 100644 --- a/lib/receipt-hunt/hunt.ts +++ b/lib/receipt-hunt/hunt.ts @@ -18,10 +18,13 @@ import { getRiskLevel } from '@/lib/pending-operations/risk-tiers' import { getMailSearchService } from '@/lib/mail-search/service' import { ingestMailCandidate } from './ingest' import { normalizeForMatch } from '@/lib/documents/core-receipt-matcher' +import { adjudicate } from './adjudicate' import { attachSekTotals } from './fx' import { extractMailDocuments } from './mail-intelligence' import { + CERTAIN_CONFIDENCE, MAX_PROPOSALS_PER_RUN, + UNCERTAIN_FLOOR, canHaveEmailReceipt, receiptIdentity, worthFetching, @@ -430,7 +433,56 @@ export async function huntCompany( } if (pool.length === 0) return base - const proposals = selectProposals(transactions, pool, suppression, limit) + // Collect the whole band the formula can speak to, then split it: what it is + // sure of goes straight through, what it is not gets a second opinion. + const scored = selectProposals(transactions, pool, suppression, limit, UNCERTAIN_FLOOR) + const certain = scored.filter((p) => p.confidence >= CERTAIN_CONFIDENCE) + const uncertain = scored.filter((p) => p.confidence < CERTAIN_CONFIDENCE) + + // Adjudicated on a dry run too. A provkörning is supposed to show exactly + // what a real run would propose, and skipping the second opinion would show + // a different, smaller answer than the one that lands. + const byIdForPairs = new Map(transactions.map((t) => [t.id, t])) + const verdicts = await adjudicate( + uncertain.map((p) => { + const tx = byIdForPairs.get(p.transaction_id) as HuntTransaction + return { + key: `${p.transaction_id}::${p.document_id}`, + purchase: { + description: tx.merchant_name || tx.description || '', + amount: Math.abs(tx.amount ?? 0), + currency: tx.currency ?? 'SEK', + date: tx.date ?? '', + }, + receipt: { + vendor: p.merchant_name, + total: p.total_amount, + currency: p.currency, + sekTotal: p.sek_total ?? null, + date: p.receipt_date, + fileName: fileNames.get(p.document_id) ?? null, + }, + confidence: p.confidence, + matchReasons: p.matchReasons, + } + }), + ) + + const accepted = new Map(verdicts.map((v) => [v.key, v.reason])) + const adjudicated = uncertain + .filter((p) => accepted.has(`${p.transaction_id}::${p.document_id}`)) + .map((p) => ({ + ...p, + // The verdict replaces the arithmetic's reasons, because it is what a + // human is being asked to check. The score stays as the formula computed + // it, unflattered: it is a real record of what the arithmetic made of + // the pair, and dressing it up would hide the very uncertainty that sent + // the pair for a second opinion. + matchReasons: [accepted.get(`${p.transaction_id}::${p.document_id}`) as string], + wasAdjudicated: true, + })) + + const proposals = [...certain, ...adjudicated] if (proposals.length === 0) return base if (dryRun) return { ...base, proposed: proposals.length, proposals } if (!userId) return { ...base, skippedNoOwner: true } @@ -469,6 +521,9 @@ export async function huntCompany( inbox_item_id: proposal.inbox_item_id, confidence: proposal.confidence, match_reasons: proposal.matchReasons, + // Which instrument decided: the arithmetic alone, or a second opinion + // on a pair the arithmetic could not settle. + decided_by: proposal.wasAdjudicated ? 'adjudicator' : 'matcher', }, } }) diff --git a/lib/receipt-hunt/select.ts b/lib/receipt-hunt/select.ts index 92a514c8..fc04cb6d 100644 --- a/lib/receipt-hunt/select.ts +++ b/lib/receipt-hunt/select.ts @@ -29,6 +29,22 @@ import { */ export const HUNT_MIN_CONFIDENCE = 0.7 +/** + * Above this the arithmetic decides alone. + * + * An exact amount from a merchant the matcher recognises needs no second + * opinion, and paying for one on every pair would be latency and cost for a + * verdict nobody doubts. + */ +export const CERTAIN_CONFIDENCE = 0.8 + +/** + * Below this nothing is worth a second opinion either: the signals disagree, + * and asking a model to rescue a pair the evidence does not support is how + * plausible-sounding wrong answers get made. + */ +export const UNCERTAIN_FLOOR = 0.6 + /** * How far clear the winner must be before we propose it. * @@ -75,6 +91,10 @@ export interface HuntProposal { receipt_date: string | null total_amount: number | null currency: string | null + /** The total in kronor, when a rate was resolved: both sides then compare. */ + sek_total?: number | null + /** Settled by a second opinion rather than by the formula alone. */ + wasAdjudicated?: boolean /** * Where the document came from, carried through from the inbox item so the * proposal can say "found in your mailbox" rather than implying a human @@ -125,6 +145,11 @@ export function selectProposals( pool: readonly HuntPoolItem[], suppression: SuppressionSets, limit: number = MAX_PROPOSALS_PER_RUN, + /** + * Lowest confidence worth returning. The caller lowers it to collect the + * band it intends to adjudicate; every other guard still applies. + */ + minConfidence: number = HUNT_MIN_CONFIDENCE, ): HuntProposal[] { if (pool.length === 0) return [] @@ -154,7 +179,7 @@ export function selectProposals( if (scored.length === 0) continue const [winner, runnerUp] = scored - if (winner.confidence < HUNT_MIN_CONFIDENCE) continue + if (winner.confidence < minConfidence) continue if (runnerUp && winner.confidence - runnerUp.confidence < AMBIGUITY_MARGIN) continue const documentId = winner.document_id as string @@ -181,6 +206,7 @@ export function selectProposals( receipt_date: winner.receipt_date, total_amount: winner.total_amount, currency: winner.currency, + sek_total: poolById.get(winner.inbox_item_id)?.sek_total ?? null, }) }