feat(receipt-hunt): let a model settle the pairs the arithmetic cannot (#1498)
The weights in the matcher are chosen by hand, and two of them were
adjusted last week because they produced a wrong answer on real data. A
pair agreeing to within 1%, from a merchant the matcher recognised, still
scored 0.62 because a date had drifted. That says more about the
constants than about the receipt.
So the formula keeps what it is good at and hands over what it is not.
Above 0.80 it decides alone: an exact amount from a recognised merchant
needs no second opinion, and buying one for every pair would be latency
and cost for a verdict nobody doubts. Below 0.60 nothing is asked either,
because inviting a model to rescue a pair the evidence does not support
is how plausible wrong answers get made. Only the band between goes for
adjudication, which on a real ledger was five pairs against one the
formula had already settled.
The question is a yes or no with a reason, never a score. An earlier
design in this feature asked a model to rate its own certainty and it
anchored on round numbers, which is what the calibration literature
predicts. Judging concrete evidence and explaining the judgement is a
different task.
A verdict is checked, not trusted: a key nobody asked about is dropped, a
pair answered twice keeps its first answer, and a failed call accepts
nothing, leaving the run exactly where the arithmetic left it. The
formula's own score is stored unflattered next to the verdict, because
dressing it up would hide the uncertainty that sent the pair for a second
opinion in the first place, and agent_metadata records which instrument
decided.
Adjudication runs on a dry run too. A provkörning is meant to show what a
real run would propose, and skipping it would show a smaller, different
answer than the one that lands.
Measured on a real ledger: five uncertain pairs asked, two accepted with
reasons a human can check ("Samma leverantör och belopp stämmer inom
rimlig valutamarginal"), three rejected. Proposals went from one to three.
Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Jakob Wennberg
Claude Opus 5
parent
4c479f84f4
commit
88140e5774
@@ -29,6 +29,22 @@ import {
|
||||
*/
|
||||
export const HUNT_MIN_CONFIDENCE = 0.7
|
||||
|
||||
/**
|
||||
* Above this the arithmetic decides alone.
|
||||
*
|
||||
* An exact amount from a merchant the matcher recognises needs no second
|
||||
* opinion, and paying for one on every pair would be latency and cost for a
|
||||
* verdict nobody doubts.
|
||||
*/
|
||||
export const CERTAIN_CONFIDENCE = 0.8
|
||||
|
||||
/**
|
||||
* Below this nothing is worth a second opinion either: the signals disagree,
|
||||
* and asking a model to rescue a pair the evidence does not support is how
|
||||
* plausible-sounding wrong answers get made.
|
||||
*/
|
||||
export const UNCERTAIN_FLOOR = 0.6
|
||||
|
||||
/**
|
||||
* How far clear the winner must be before we propose it.
|
||||
*
|
||||
@@ -75,6 +91,10 @@ export interface HuntProposal {
|
||||
receipt_date: string | null
|
||||
total_amount: number | null
|
||||
currency: string | null
|
||||
/** The total in kronor, when a rate was resolved: both sides then compare. */
|
||||
sek_total?: number | null
|
||||
/** Settled by a second opinion rather than by the formula alone. */
|
||||
wasAdjudicated?: boolean
|
||||
/**
|
||||
* Where the document came from, carried through from the inbox item so the
|
||||
* proposal can say "found in your mailbox" rather than implying a human
|
||||
@@ -125,6 +145,11 @@ export function selectProposals(
|
||||
pool: readonly HuntPoolItem[],
|
||||
suppression: SuppressionSets,
|
||||
limit: number = MAX_PROPOSALS_PER_RUN,
|
||||
/**
|
||||
* Lowest confidence worth returning. The caller lowers it to collect the
|
||||
* band it intends to adjudicate; every other guard still applies.
|
||||
*/
|
||||
minConfidence: number = HUNT_MIN_CONFIDENCE,
|
||||
): HuntProposal[] {
|
||||
if (pool.length === 0) return []
|
||||
|
||||
@@ -154,7 +179,7 @@ export function selectProposals(
|
||||
if (scored.length === 0) continue
|
||||
|
||||
const [winner, runnerUp] = scored
|
||||
if (winner.confidence < HUNT_MIN_CONFIDENCE) continue
|
||||
if (winner.confidence < minConfidence) continue
|
||||
if (runnerUp && winner.confidence - runnerUp.confidence < AMBIGUITY_MARGIN) continue
|
||||
|
||||
const documentId = winner.document_id as string
|
||||
@@ -181,6 +206,7 @@ export function selectProposals(
|
||||
receipt_date: winner.receipt_date,
|
||||
total_amount: winner.total_amount,
|
||||
currency: winner.currency,
|
||||
sek_total: poolById.get(winner.inbox_item_id)?.sek_total ?? null,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user