From 4c479f84f4c1eed9a38e8375d78f30f31a596c9c Mon Sep 17 00:00:00 2001 From: Jakob Wennberg Date: Mon, 10 Aug 2026 17:54:22 +0200 Subject: [PATCH] feat(receipt-hunt): convert a foreign receipt instead of refusing to compare it (#1497) * feat(receipt-hunt): compare a foreign receipt by converting it, not by refusing A Swedish bank posts a converted figure for a purchase abroad while the receipt states the original: Anthropic bills 180,00 EUR and the statement reads -2 014,32 kr. Neither number appears in the other document, so the matcher refused the pair rather than guess. On a SaaS-heavy ledger that is not an edge case: of 25 receipts fetched from a real company's mailboxes, 14 were in USD or EUR and none could ever pair. The receipt's total is now resolved into kronor with Riksbanken's rate for its own date, and handed to the same matcher, which still wants the merchant and the date to agree. The seam already existed: the scorer passed null where a SEK value would go, with a comment explaining that cross-currency pairs were deliberately incomparable. Surfaces that do not resolve a rate still pass nothing and behave exactly as before. Rates are fetched once per currency and day. Riksbanken answers 429 to a caller that asks per document, and a run holds a dozen receipts from one vendor in one month. A rate that cannot be resolved leaves the receipt exactly as incomparable as it was. Two calibration faults surfaced once the amounts became comparable: A converted total is judged at 9% rather than 5%. Riksbanken publishes a mid rate and a card issuer charges its own, so the two carry a known spread on top of any disagreement about the sum: measured against real statements, 1.2% to 3%. Holding both to one bar treats a rate spread as if it were a discrepancy. The date tolerance moves from 3 days to 10. A card settles days after the purchase, an international one routinely a week later, and a forwarded receipt carries the purchase date while the statement carries the posting. At three days the signal scored zero for ordinary correct pairs and took a quarter of the weight with it: a receipt agreeing to within 1%, from a merchant the matcher recognised, still capped at 0.62. The 27 human- confirmed pairs and the 7 near-misses that must not match all still hold. Measured on that ledger: purchases with any candidate at all go 9 -> 13, and the Anthropic pair proposes at 0.82 where it was previously unscoreable. Co-Authored-By: Claude Opus 5 (1M context) * fix(receipt-hunt): an undated receipt is not converted at today's rate Raised in review. A foreign receipt with no invoiceDate fell back to the current date, which put a receipt of unknown age into amount matching on the strength of a guess: a rate two years out is how something incomparable acquires confidence it has not earned. No date, no conversion, and the receipt stays exactly as it was. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) --- lib/agent-context/underlag-candidates.ts | 22 +++- lib/documents/core-receipt-matcher.ts | 29 +++++- lib/receipt-hunt/__tests__/fx.test.ts | 113 +++++++++++++++++++++ lib/receipt-hunt/fx.ts | 123 +++++++++++++++++++++++ lib/receipt-hunt/hunt.ts | 9 +- lib/receipt-hunt/select.ts | 2 + 6 files changed, 292 insertions(+), 6 deletions(-) create mode 100644 lib/receipt-hunt/__tests__/fx.test.ts create mode 100644 lib/receipt-hunt/fx.ts diff --git a/lib/agent-context/underlag-candidates.ts b/lib/agent-context/underlag-candidates.ts index b892c503..a8a5d42b 100644 --- a/lib/agent-context/underlag-candidates.ts +++ b/lib/agent-context/underlag-candidates.ts @@ -23,6 +23,7 @@ */ import type { SupabaseClient } from '@supabase/supabase-js' import { + CONVERTED_AMOUNT_TOLERANCE_PERCENT, amountVarianceForMatch, calculateMatchConfidence, calculateMerchantSimilarity, @@ -84,6 +85,14 @@ interface ScorableItem { document_id: string | null extracted_data: InvoiceExtractionResult | null channel_context: InboxChannelContext | null + /** + * The receipt's total in kronor, when a caller has resolved a rate for it. + * + * Left undefined by every surface that has not, which keeps the old + * behaviour exactly: a cross-currency pair stays incomparable rather than + * being scored on date and merchant alone. + */ + sek_total?: number | null } /** Pull the fields the matcher needs out of an extraction blob. */ @@ -126,10 +135,10 @@ export function scoreUnderlagCandidates( const amountVariance = amountVarianceForMatch( sig.total, sig.currency, - // No stored SEK value on the inbox item, so cross-currency pairs are - // deliberately not comparable and the matcher drops the amount signal - // rather than matching 750 EUR to 750 SEK. - null, + // A SEK value only when someone resolved a rate for this receipt. + // Without one the pair stays incomparable, rather than matching 750 EUR + // to 750 SEK. + item.sek_total ?? null, tx.amount, txCurrency, txSek, @@ -150,10 +159,15 @@ export function scoreUnderlagCandidates( : Number.POSITIVE_INFINITY const similarity = sig.supplier ? calculateMerchantSimilarity(sig.supplier, txMerchant) : 0 + // A converted total is judged against the wider bar, because the rate + // spread is a known error rather than a disagreement about the sum. + const converted = sig.currency !== txCurrency && item.sek_total != null const { confidence, matchReasons } = calculateMatchConfidence( dateVariance, amountVariance, similarity, + undefined, + converted ? CONVERTED_AMOUNT_TOLERANCE_PERCENT : undefined, ) if (confidence < CANDIDATE_MIN_CONFIDENCE) continue diff --git a/lib/documents/core-receipt-matcher.ts b/lib/documents/core-receipt-matcher.ts index c3e26028..42c3a675 100644 --- a/lib/documents/core-receipt-matcher.ts +++ b/lib/documents/core-receipt-matcher.ts @@ -6,8 +6,35 @@ */ // Matching configuration (re-exported for consumers) -export const DATE_TOLERANCE_DAYS = 3 +/** + * How far a receipt's date may sit from the bank's before the date stops + * counting as agreement. + * + * Ten days, not three. A card purchase settles days after it happens, an + * international one routinely a week later, and a forwarded receipt carries + * the date of the purchase while the statement carries the date of the + * posting. At three days the signal scored zero for ordinary, correct pairs + * and took a quarter of the weight down with it: a receipt matching to within + * 1% from a merchant the matcher recognised still capped at 0.62, under every + * threshold that decides anything. + * + * The date remains real evidence at this width. A receipt from March still + * disagrees with a purchase in September. + */ +export const DATE_TOLERANCE_DAYS = 10 export const AMOUNT_TOLERANCE_PERCENT = 0.05 + +/** + * Tolerance for a total that had to be converted into kronor first. + * + * A same-currency comparison is two readings of one number, so 5% is generous. + * A converted one carries a second, known error: Riksbanken publishes a mid + * rate and a card issuer charges its own, typically a point or two away, on a + * settlement day that need not be the receipt's. Holding both to the same bar + * treats a rate spread as if it were a disagreement about the sum. Measured + * against real statements, the spread ran 1.2% to 3%. + */ +export const CONVERTED_AMOUNT_TOLERANCE_PERCENT = 0.09 export const MIN_MATCH_CONFIDENCE = 0.4 /** diff --git a/lib/receipt-hunt/__tests__/fx.test.ts b/lib/receipt-hunt/__tests__/fx.test.ts new file mode 100644 index 00000000..64a567f1 --- /dev/null +++ b/lib/receipt-hunt/__tests__/fx.test.ts @@ -0,0 +1,113 @@ +/** + * Putting a foreign receipt into kronor. What matters is that it only ever + * adds a number nobody had, never removes a pair the hunt could already make, + * and never invents a rate it could not fetch. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { roundOre } from '@/lib/money' +import type { HuntPoolItem } from '../select' + +const mockFetchRate = vi.fn() +vi.mock('@/lib/currency/riksbanken', () => ({ + fetchExchangeRate: (...args: unknown[]) => mockFetchRate(...args), + convertToSEK: (amount: number, rate: number) => amount * rate, +})) + +import { attachSekTotals } from '../fx' + +function item( + currency: string, + total: number | null, + date: string | null = '2026-06-15', + id = 'i1', +): HuntPoolItem { + return { + id, + document_id: `d-${id}`, + channel_context: null, + extracted_data: { + supplier: { name: 'Anthropic, PBC' }, + invoice: { currency, invoiceDate: date }, + totals: { total }, + }, + } +} + +beforeEach(() => { + vi.clearAllMocks() + mockFetchRate.mockResolvedValue({ currency: 'EUR', rate: 11.19, date: '2026-06-15' }) +}) + +describe('attachSekTotals', () => { + it('resolves a foreign total into kronor', async () => { + // Anthropic bills 180 EUR; the statement reads -2 014,32 kr. Neither + // number appears in the other document. + const [out] = await attachSekTotals({} as never, [item('EUR', 180)]) + expect(out.sek_total).toBe(2014.2) + }) + + it('rounds to öre rather than carrying a float into a comparison', async () => { + mockFetchRate.mockResolvedValue({ currency: 'USD', rate: 9.747, date: '2026-06-19' }) + const [out] = await attachSekTotals({} as never, [item('USD', 104.23)]) + expect(out.sek_total).toBe(roundOre(104.23 * 9.747)) + }) + + it('leaves a Swedish receipt alone', async () => { + const [out] = await attachSekTotals({} as never, [item('SEK', 425)]) + expect(out.sek_total).toBeUndefined() + expect(mockFetchRate).not.toHaveBeenCalled() + }) + + it('asks for a rate once per currency and day, not once per receipt', async () => { + // Riksbanken answers 429 to a caller that asks per document, and a run + // holds a dozen receipts from the same vendor in the same month. + await attachSekTotals({} as never, [ + item('EUR', 180, '2026-06-15', 'a'), + item('EUR', 225, '2026-06-15', 'b'), + item('EUR', 22.5, '2026-06-15', 'c'), + ]) + expect(mockFetchRate).toHaveBeenCalledTimes(1) + }) + + it('still asks again for another day', async () => { + await attachSekTotals({} as never, [ + item('EUR', 180, '2026-06-15', 'a'), + item('EUR', 180, '2026-07-15', 'b'), + ]) + expect(mockFetchRate).toHaveBeenCalledTimes(2) + }) + + it('leaves the receipt untouched when no rate can be had', async () => { + // Exactly as incomparable as before, which is the point: the hunt loses + // nothing it previously had. + mockFetchRate.mockResolvedValue(null) + const [out] = await attachSekTotals({} as never, [item('EUR', 180)]) + expect(out.sek_total).toBeUndefined() + }) + + it('survives the rate service failing outright', async () => { + mockFetchRate.mockRejectedValue(new Error('riksbanken 429')) + const [out] = await attachSekTotals({} as never, [item('EUR', 180)]) + expect(out.sek_total).toBeUndefined() + }) + + it('does not guess at a currency Riksbanken has no series for', async () => { + const [out] = await attachSekTotals({} as never, [item('ZWL', 500)]) + expect(out.sek_total).toBeUndefined() + expect(mockFetchRate).not.toHaveBeenCalled() + }) + + it('refuses to date an undated receipt with today', async () => { + // Today's rate on a receipt of unknown age would make something + // incomparable look comparable, which is how a wrong pairing gets + // confidence it has not earned. + const [out] = await attachSekTotals({} as never, [item('EUR', 180, null)]) + expect(out.sek_total).toBeUndefined() + }) + + it('ignores a receipt with no total to convert', async () => { + const [out] = await attachSekTotals({} as never, [item('EUR', null)]) + expect(out.sek_total).toBeUndefined() + expect(mockFetchRate).not.toHaveBeenCalled() + }) +}) diff --git a/lib/receipt-hunt/fx.ts b/lib/receipt-hunt/fx.ts new file mode 100644 index 00000000..893e18ee --- /dev/null +++ b/lib/receipt-hunt/fx.ts @@ -0,0 +1,123 @@ +/** + * Putting a foreign receipt into kronor so it can be compared at all. + * + * Swedish banks post a converted SEK figure for a card purchase abroad, and + * the receipt states the original: Anthropic bills 180,00 EUR and the + * statement reads -2 014,32 kr. Neither number appears in the other document, + * so the matcher has always refused the pair rather than guess. On a + * SaaS-heavy ledger that is not an edge case: measured on a real company, 14 + * of 25 fetched receipts were in USD or EUR and none of them could ever pair. + * + * The conversion is deliberately not a match on its own. It fills in the one + * missing number and hands the pair back to the same matcher, which still + * wants the merchant and the date to agree, and still applies its own + * tolerance. That tolerance is what absorbs the difference between + * Riksbanken's mid rate and what a card issuer actually charged: measured + * against real statements, Supabase came out 1.22% off and Vercel 1.23%, + * comfortably inside the 5% the matcher already allows. + */ +import type { SupabaseClient } from '@supabase/supabase-js' +import { convertToSEK, fetchExchangeRate } from '@/lib/currency/riksbanken' +import { roundOre } from '@/lib/money' +import { createLogger } from '@/lib/logger' +import type { HuntPoolItem } from './select' + +const log = createLogger('receipt-hunt-fx') + +/** Currencies Riksbanken publishes a series for. */ +const SUPPORTED = new Set(['EUR', 'USD', 'GBP', 'NOK', 'DKK', 'CHF', 'JPY', 'PLN']) + +interface Signals { + currency: string + total: number | null + date: string | null +} + +function signalsOf(item: HuntPoolItem): Signals { + const data = item.extracted_data as + | { + invoice?: { currency?: string | null; invoiceDate?: string | null } + totals?: { total?: number | null } + } + | null + | undefined + return { + currency: (data?.invoice?.currency || 'SEK').toUpperCase(), + total: data?.totals?.total ?? null, + date: data?.invoice?.invoiceDate ?? null, + } +} + +/** + * Resolve a SEK total for every pool item stated in another currency. + * + * Rates are fetched once per currency and day and reused, because Riksbanken + * answers 429 to a caller that asks per document, and a run can hold a dozen + * receipts from the same vendor in the same month. + * + * An item whose rate cannot be resolved is returned untouched, which leaves it + * exactly as incomparable as it was before: the hunt loses nothing it had. + */ +export async function attachSekTotals( + supabase: SupabaseClient, + pool: readonly HuntPoolItem[], +): Promise { + const rates = new Map() + + const rateFor = async (currency: string, date: string | null): Promise => { + // No date, no conversion. Reaching for today's rate would put a receipt + // whose age nobody knows into amount matching on the strength of a guess, + // and a rate two years out is how an incomparable receipt becomes a + // confident wrong pairing. + if (!date) return null + + // Riksbanken publishes per day, so the day is the whole cache key. + const day = date + const key = `${currency}::${day}` + if (rates.has(key)) return rates.get(key) ?? null + + try { + const rate = await fetchExchangeRate(currency as never, new Date(day), supabase as never) + rates.set(key, rate?.rate ?? null) + return rate?.rate ?? null + } catch (error) { + log.warn('could not resolve an exchange rate', { + currency, + day, + error: error instanceof Error ? error.message : String(error), + }) + rates.set(key, null) + return null + } + } + + const out: HuntPoolItem[] = [] + let converted = 0 + + for (const item of pool) { + const sig = signalsOf(item) + if (sig.currency === 'SEK' || sig.total == null || !SUPPORTED.has(sig.currency)) { + out.push(item) + continue + } + + const rate = await rateFor(sig.currency, sig.date) + if (rate == null) { + out.push(item) + continue + } + + converted++ + out.push({ + ...item, + // Öre, like every other money value here: a raw float would put + // 1015.9700000000001 into a comparison against a bank amount. + sek_total: roundOre(convertToSEK(sig.total, rate)), + }) + } + + if (converted > 0) { + log.info('resolved foreign receipt totals', { converted, rates: rates.size }) + } + return out +} diff --git a/lib/receipt-hunt/hunt.ts b/lib/receipt-hunt/hunt.ts index a7586f4b..66ef967c 100644 --- a/lib/receipt-hunt/hunt.ts +++ b/lib/receipt-hunt/hunt.ts @@ -18,6 +18,7 @@ import { getRiskLevel } from '@/lib/pending-operations/risk-tiers' import { getMailSearchService } from '@/lib/mail-search/service' import { ingestMailCandidate } from './ingest' import { normalizeForMatch } from '@/lib/documents/core-receipt-matcher' +import { attachSekTotals } from './fx' import { extractMailDocuments } from './mail-intelligence' import { MAX_PROPOSALS_PER_RUN, @@ -412,7 +413,13 @@ export async function huntCompany( } } - const { pool, fileNames } = await fetchPool(supabase, companyId) + const { pool: rawPool, fileNames } = await fetchPool(supabase, companyId) + + // A receipt in USD or EUR carries a number the bank statement never shows. + // Resolving it into kronor is what lets the ordinary matcher weigh the + // amount at all; without it those pairs are refused, which on a SaaS-heavy + // ledger is most of them. + const pool = await attachSekTotals(supabase, rawPool) const base: HuntCompanyResult = { companyId, diff --git a/lib/receipt-hunt/select.ts b/lib/receipt-hunt/select.ts index 316ced5b..92a514c8 100644 --- a/lib/receipt-hunt/select.ts +++ b/lib/receipt-hunt/select.ts @@ -60,6 +60,8 @@ export interface HuntPoolItem { document_id: string | null extracted_data: unknown channel_context: unknown + /** The receipt's total in kronor, resolved from a rate before scoring. */ + sek_total?: number | null } export interface HuntProposal {