fix(categorization): connect card descriptors to counterparty history (#1095)
* fix(categorization): connect card descriptors to counterparty history
suggest_categories returned no signal for recurring card merchants
(reported: Anthropic booked to 5420 fourteen times, zero suggestions).
Three compounding causes, all fixed:
- normalizeCounterpartyName() now reduces card-network descriptors to
their merchant segment ("ANTHROPIC* CLAUDE SUB SAN FRANCISCO" ->
"anthropic"; "PAYPAL *SPOTIFY" -> "spotify"), so monthly per-charge
tails stop splintering one merchant into unmatchable variants. SQL
mirror normalize_counterparty_key() updated in lockstep (migration
20260721140000), keeping the ledger-context template join exact.
- New token_subset match tier bridges templates learned from manual
bookings ("Claude Dec" -> "claude") to bank descriptors containing
the token, and card-core descriptors to legacy splintered templates.
Guarded by a distinctive-token filter so generic/geo words never
match on their own.
- Merchant history falls back to description when merchant_name is
null: card purchases never carry merchant_name, so the history path
was structurally blind to exactly the transactions that need it.
History keys now share the counterparty-template normalization and
the 200-row window is ordered by recency.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
* fix(categorization): guard single-token matches, anchor history on original_description
Review follow-ups (CodeRabbit on #1095):
- token_subset tier: a single shared distinctive token now also requires
occurrence_count >= 3 on the template, so a template named after a
common word or first name (one prior booking) cannot vacuum up
unrelated transfers ("SWISH ANDERS JOHANSSON"). Multi-token agreement
stays unrestricted; the Claude/Anthropic case (14 bookings) is
unaffected.
- merchant history keys on original_description ?? description: the raw
bank descriptor is immutable while description is a user-editable
working title, so renaming a transaction no longer severs its history
link for future recurring charges.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
* chore(migrations): re-timestamp card-descriptor migration after prod moved past it
Prod applied 20260721144311 (#1101) through 20260721201747 (#1104) while
this PR was open; 20260721140000 would sort before them and risk being
skipped by out-of-order auto-apply at merge. Not yet applied to prod, so
renaming is safe; the preview branch re-applies idempotently
(CREATE OR REPLACE).
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---------
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -270,3 +270,4 @@ One line per decision: `[YYYY-MM-DD] <decision>: <why>`. Appended by agents and
|
||||
[2026-07-21] Fortnox/SIE atomic import timeout fixed by function-scoped statement_timeout (290s) on import_sie_journal_entries, not by chunking the RPC: chunking would reintroduce the partial-import states the atomic RPC exists to eliminate (20260712150000), and the 20260629160100 delete-path precedent already uses the same bound; pg-real ratchet pins the config on all three SIE RPCs because CREATE OR REPLACE silently drops ALTER FUNCTION settings.
|
||||
[2026-07-21] Keep live annual-report narrative editable after a version is locked: immutable version snapshots preserve signing and filing content, while edits must remain possible to create a corrected superseding version.
|
||||
[2026-07-21] Restrict annual-report signature evidence transitions to the server service role and structured opaque references: browser RLS may manage only unbound pending roster rows, so route validation cannot be bypassed and evidence references cannot carry free-text personal data.
|
||||
[2026-07-21] Card-descriptor normalization keys on the pre-star merchant segment (post-star for processor prefixes) plus a token_subset match tier, instead of the deferred AI descriptor normalization (data_quality_master Appendix B): deterministic, mirrors into normalize_counterparty_key() so ledger-context template joins stay exact, and fixes the reported Anthropic no-signal case with no new infrastructure. Merchant history now falls back to description because card purchases never carry merchant_name.
|
||||
|
||||
@@ -44,11 +44,12 @@ export const POST = withRouteContext(
|
||||
// (global frequency padding produced identical low-confidence spreads).
|
||||
const { data: historicalTxns } = await supabase
|
||||
.from('transactions')
|
||||
.select('category, merchant_name')
|
||||
.select('category, merchant_name, description, original_description')
|
||||
.eq('company_id', companyId)
|
||||
.not('is_business', 'is', null)
|
||||
.neq('category', 'uncategorized')
|
||||
.neq('category', 'private')
|
||||
.order('date', { ascending: false })
|
||||
.limit(200)
|
||||
|
||||
const merchantHistory = buildMerchantHistory(historicalTxns ?? [])
|
||||
@@ -72,7 +73,11 @@ export const POST = withRouteContext(
|
||||
suggestions[tx.id] = getSuggestedCategories(
|
||||
tx as Transaction,
|
||||
mappingRules || [],
|
||||
merchantHistoryFor(merchantHistory, (tx as Transaction).merchant_name)
|
||||
merchantHistoryFor(
|
||||
merchantHistory,
|
||||
(tx as Transaction).merchant_name,
|
||||
(tx as Transaction).original_description ?? (tx as Transaction).description,
|
||||
)
|
||||
)
|
||||
template_suggestions[tx.id] = await getSuggestedTemplates(tx as Transaction, entityType, mappingRules || undefined)
|
||||
}
|
||||
|
||||
@@ -5152,11 +5152,12 @@ export const tools: McpTool[] = [
|
||||
// ~0.5 four-way spread agents reported as pure noise (P2-1).
|
||||
const { data: historicalTxns } = await supabase
|
||||
.from('transactions')
|
||||
.select('category, merchant_name')
|
||||
.select('category, merchant_name, description, original_description')
|
||||
.eq('company_id', companyId)
|
||||
.not('is_business', 'is', null)
|
||||
.neq('category', 'uncategorized')
|
||||
.neq('category', 'private')
|
||||
.order('date', { ascending: false })
|
||||
.limit(200)
|
||||
|
||||
const merchantHistory = buildMerchantHistory(historicalTxns ?? [])
|
||||
@@ -5173,7 +5174,11 @@ export const tools: McpTool[] = [
|
||||
for (const tx of transactions) {
|
||||
suggestions[tx.id] = getSuggestedCategories(
|
||||
tx as Transaction, mappingRules ?? [],
|
||||
merchantHistoryFor(merchantHistory, (tx as Transaction).merchant_name)
|
||||
merchantHistoryFor(
|
||||
merchantHistory,
|
||||
(tx as Transaction).merchant_name,
|
||||
(tx as Transaction).original_description ?? (tx as Transaction).description,
|
||||
)
|
||||
)
|
||||
|
||||
const cpMatch = counterpartyMatches.get(tx.id)
|
||||
|
||||
@@ -90,6 +90,21 @@ describe('counterparty-templates', () => {
|
||||
expect(normalizeCounterpartyName('NORDEA SEB')).toBe('nordea seb') // SEB is 3 chars: kept
|
||||
expect(normalizeCounterpartyName('KLARNA')).toBe('klarna') // single token kept
|
||||
})
|
||||
|
||||
it('reduces card-network descriptors to the merchant segment', () => {
|
||||
// The Anthropic splinter bug: the per-charge tail after '*' (product,
|
||||
// ref, city) changes between charges, so keeping it makes every
|
||||
// recurring foreign SaaS subscription a new counterparty each month.
|
||||
expect(normalizeCounterpartyName('ANTHROPIC* CLAUDE SUB SAN FRANCISCO')).toBe('anthropic')
|
||||
expect(normalizeCounterpartyName('ANTHROPIC*CLAUDE SUB +14155551234')).toBe('anthropic')
|
||||
expect(normalizeCounterpartyName('AMZN MKTP SE*A12B34CD5')).toBe('amzn mktp')
|
||||
})
|
||||
|
||||
it('keeps the merchant AFTER the star for processor descriptors', () => {
|
||||
expect(normalizeCounterpartyName('PAYPAL *SPOTIFY')).toBe('spotify')
|
||||
expect(normalizeCounterpartyName('SQ *BLUE BOTTLE COFFEE')).toBe('blue bottle coffee')
|
||||
expect(normalizeCounterpartyName('KLARNA*BOOZT FASHION')).toBe('boozt fashion')
|
||||
})
|
||||
})
|
||||
|
||||
// ── Confidence ─────────────────────────────────────────────
|
||||
@@ -219,6 +234,132 @@ describe('counterparty-templates', () => {
|
||||
|
||||
expect(result).toBeNull()
|
||||
})
|
||||
|
||||
it('token-subset matches a template token buried in a card descriptor', async () => {
|
||||
// The reported Anthropic case: a template learned from manual bookings
|
||||
// ("Claude Dec" -> "claude") must match the bank's card descriptor even
|
||||
// though the strings differ by whole words, not typos.
|
||||
const template = makeCategorizationTemplate({
|
||||
counterparty_name: 'claude',
|
||||
confidence: 0.8,
|
||||
counterparty_aliases: ['claude dec'],
|
||||
})
|
||||
const { supabase, enqueue } = createQueuedMockSupabase()
|
||||
enqueue({ data: [template] })
|
||||
|
||||
const tx = makeTransaction({
|
||||
merchant_name: null,
|
||||
description: 'ANTHROPIC* CLAUDE SUB SAN FRANCISCO',
|
||||
})
|
||||
const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx)
|
||||
|
||||
expect(result).not.toBeNull()
|
||||
expect(result!.matchMethod).toBe('token_subset')
|
||||
expect(result!.confidence).toBeCloseTo(0.68, 2) // 0.8 * 0.85
|
||||
})
|
||||
|
||||
it('token-subset matches a card-core descriptor against a legacy splintered template', async () => {
|
||||
// Templates learned before card-core normalization stored the full
|
||||
// descriptor; the new normalized core must still find them.
|
||||
const template = makeCategorizationTemplate({
|
||||
counterparty_name: 'anthropic claude sub san francisco',
|
||||
confidence: 0.7,
|
||||
counterparty_aliases: [],
|
||||
})
|
||||
const { supabase, enqueue } = createQueuedMockSupabase()
|
||||
enqueue({ data: [template] })
|
||||
|
||||
const tx = makeTransaction({
|
||||
merchant_name: null,
|
||||
description: 'ANTHROPIC*CLAUDE SUB LONDON',
|
||||
})
|
||||
const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx)
|
||||
|
||||
expect(result).not.toBeNull()
|
||||
expect(result!.matchMethod).toBe('token_subset')
|
||||
})
|
||||
|
||||
it('suppresses single-token matches without booking history behind them', async () => {
|
||||
// A template named after a common first name (one prior booking) must
|
||||
// not vacuum up an unrelated SWISH transfer that shares the name.
|
||||
const template = makeCategorizationTemplate({
|
||||
counterparty_name: 'anders',
|
||||
confidence: 0.5,
|
||||
occurrence_count: 1,
|
||||
counterparty_aliases: [],
|
||||
})
|
||||
const { supabase, enqueue } = createQueuedMockSupabase()
|
||||
enqueue({ data: [template] })
|
||||
|
||||
const tx = makeTransaction({ merchant_name: null, description: 'SWISH ANDERS JOHANSSON' })
|
||||
const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx)
|
||||
|
||||
expect(result).toBeNull()
|
||||
})
|
||||
|
||||
it('multi-token agreement matches without an occurrence floor', async () => {
|
||||
const template = makeCategorizationTemplate({
|
||||
counterparty_name: 'blue bottle',
|
||||
confidence: 0.5,
|
||||
occurrence_count: 1,
|
||||
counterparty_aliases: [],
|
||||
})
|
||||
const { supabase, enqueue } = createQueuedMockSupabase()
|
||||
enqueue({ data: [template] })
|
||||
|
||||
const tx = makeTransaction({
|
||||
merchant_name: null,
|
||||
description: 'SQ *BLUE BOTTLE COFFEE OAKLAND',
|
||||
})
|
||||
const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx)
|
||||
|
||||
expect(result).not.toBeNull()
|
||||
expect(result!.matchMethod).toBe('token_subset')
|
||||
})
|
||||
|
||||
it('never token-matches on generic tokens alone', async () => {
|
||||
// A template whose name is all generic/geo words must not vacuum up
|
||||
// every descriptor mentioning them.
|
||||
const template = makeCategorizationTemplate({
|
||||
counterparty_name: 'sverige',
|
||||
confidence: 0.9,
|
||||
counterparty_aliases: [],
|
||||
})
|
||||
const { supabase, enqueue } = createQueuedMockSupabase()
|
||||
enqueue({ data: [template] })
|
||||
|
||||
const tx = makeTransaction({ merchant_name: 'IKEA SVERIGE STOCKHOLM' })
|
||||
const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx)
|
||||
|
||||
expect(result).toBeNull()
|
||||
})
|
||||
|
||||
it('prefers exact normalized over token subset', async () => {
|
||||
const exact = makeCategorizationTemplate({
|
||||
id: 'tmpl-exact',
|
||||
counterparty_name: 'anthropic',
|
||||
confidence: 0.8,
|
||||
counterparty_aliases: [],
|
||||
})
|
||||
const tokenish = makeCategorizationTemplate({
|
||||
id: 'tmpl-token',
|
||||
counterparty_name: 'claude',
|
||||
confidence: 0.8,
|
||||
counterparty_aliases: [],
|
||||
})
|
||||
const { supabase, enqueue } = createQueuedMockSupabase()
|
||||
enqueue({ data: [tokenish, exact] })
|
||||
|
||||
const tx = makeTransaction({
|
||||
merchant_name: null,
|
||||
description: 'ANTHROPIC* CLAUDE SUB SAN FRANCISCO',
|
||||
})
|
||||
const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx)
|
||||
|
||||
expect(result).not.toBeNull()
|
||||
expect(result!.matchMethod).toBe('exact_normalized')
|
||||
expect(result!.template.id).toBe('tmpl-exact')
|
||||
})
|
||||
})
|
||||
|
||||
// ── Build MappingResult ────────────────────────────────────
|
||||
|
||||
@@ -64,15 +64,9 @@ function stripTrailingNoiseTokens(s: string): string {
|
||||
return tokens.join(' ')
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a transaction description to a canonical counterparty name.
|
||||
*
|
||||
* Strips bank transfer prefixes, trailing dates, invoice references, trailing
|
||||
* digit sequences, and trailing period/initials tokens, then delegates to
|
||||
* normalizeMerchantName() for Swedish company suffix removal and lowercasing.
|
||||
*/
|
||||
export function normalizeCounterpartyName(raw: string): string {
|
||||
const cleaned = raw
|
||||
/** Shared first stage: bank-feed noise that is never merchant identity. */
|
||||
function stripBankNoise(raw: string): string {
|
||||
return raw
|
||||
// Strip common bank transfer prefixes
|
||||
.replace(/^(BANKGIRO|SWISH|KORTKÖP|KORT\s*KÖP|PG|BG|AUTOGIRO|PLUSGIRO)\s*/i, '')
|
||||
// Strip dates (20240615, 2024-06-15, 24-06-15)
|
||||
@@ -83,12 +77,95 @@ export function normalizeCounterpartyName(raw: string): string {
|
||||
// Strip trailing sequences of 4+ digits (card numbers, transaction refs)
|
||||
.replace(/\s+\d{4,}\s*$/g, '')
|
||||
.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Payment processors whose card descriptors put the real merchant AFTER the
|
||||
* star ("PAYPAL *SPOTIFY", "SQ *BLUE BOTTLE"). For every other star
|
||||
* descriptor the merchant is the segment before it.
|
||||
*/
|
||||
const PROCESSOR_STAR_PREFIXES = new Set([
|
||||
'paypal', 'klarna', 'izettle', 'zettle', 'iz', 'sq', 'sp', 'sumup',
|
||||
'google', 'stripe', 'payu', 'mollie',
|
||||
])
|
||||
|
||||
/**
|
||||
* Reduce a card-network descriptor to its merchant segment. Card descriptors
|
||||
* embed '*' between merchant identity and a per-charge tail (product, order
|
||||
* ref, city): "ANTHROPIC* CLAUDE SUB SAN FRANCISCO" is the merchant
|
||||
* "ANTHROPIC", not five words. The tail varies between charges, so keeping it
|
||||
* splinters every recurring foreign SaaS subscription into a new counterparty
|
||||
* each month. No '*' → returned unchanged.
|
||||
*
|
||||
* Mirrored in SQL by normalize_counterparty_key(); keep the two in sync
|
||||
* (tests/pg/ledger-usage-stats-rpc.pg.test.ts asserts parity).
|
||||
*/
|
||||
function extractCardDescriptorCore(cleaned: string): string {
|
||||
const starIdx = cleaned.indexOf('*')
|
||||
if (starIdx === -1) return cleaned
|
||||
const head = cleaned.slice(0, starIdx).trim()
|
||||
const tail = cleaned.slice(starIdx + 1).trim()
|
||||
const headKey = head.toLowerCase().replace(/[^a-z0-9åäöé]/g, '')
|
||||
if (PROCESSOR_STAR_PREFIXES.has(headKey) && tail) return tail
|
||||
if (headKey.length >= 3) return head
|
||||
return tail || head
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a transaction description to a canonical counterparty name.
|
||||
*
|
||||
* Strips bank transfer prefixes, trailing dates, invoice references, trailing
|
||||
* digit sequences, card-descriptor tails, and trailing period/initials tokens,
|
||||
* then delegates to normalizeMerchantName() for Swedish company suffix removal
|
||||
* and lowercasing.
|
||||
*/
|
||||
export function normalizeCounterpartyName(raw: string): string {
|
||||
const cleaned = extractCardDescriptorCore(stripBankNoise(raw))
|
||||
|
||||
// Drop trailing month/initials tokens before merchant-name normalization so
|
||||
// "ngrok JW" and "Ngrok Mars" collapse to the same canonical "ngrok".
|
||||
return normalizeMerchantName(stripTrailingNoiseTokens(cleaned))
|
||||
}
|
||||
|
||||
/**
|
||||
* Full normalized token set of a descriptor WITHOUT the card-core reduction:
|
||||
* the product segment of a card descriptor ("CLAUDE SUB") often carries the
|
||||
* very token an existing template is named by ("claude", learned from manual
|
||||
* bookings of the same subscription). Feeds the token_subset match tier only;
|
||||
* canonical identity stays normalizeCounterpartyName().
|
||||
*/
|
||||
export function counterpartyTokenSet(raw: string): Set<string> {
|
||||
const full = normalizeMerchantName(stripBankNoise(raw))
|
||||
return new Set(full.split(' ').filter(Boolean))
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokens too generic to identify a merchant on their own: month labels,
|
||||
* commerce noise, and geo words that ride along on bank descriptors.
|
||||
*/
|
||||
const GENERIC_TOKENS = new Set([
|
||||
...TRAILING_MONTH_TOKENS,
|
||||
'subscription', 'subscr', 'abonnemang', 'betalning', 'payment', 'purchase',
|
||||
'online', 'store', 'shop', 'butik', 'faktura', 'invoice',
|
||||
'sweden', 'sverige', 'stockholm', 'göteborg', 'goteborg', 'malmö', 'malmo',
|
||||
])
|
||||
|
||||
/** Distinctive = long and specific enough to identify a merchant. */
|
||||
function distinctiveTokens(tokens: string[]): string[] {
|
||||
return tokens.filter(
|
||||
(t) => t.length >= 4 && !GENERIC_TOKENS.has(t) && !/^\d+$/.test(t)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* A single shared token is thin evidence: require the template to be backed
|
||||
* by real booking history before trusting it, so a template named after a
|
||||
* common word or first name (template "anders", occurrence 1) cannot vacuum
|
||||
* up unrelated transfers ("SWISH ANDERS JOHANSSON"). Multi-token agreement
|
||||
* is specific enough on its own.
|
||||
*/
|
||||
const MIN_SINGLE_TOKEN_OCCURRENCES = 3
|
||||
|
||||
// ── Confidence ─────────────────────────────────────────────────
|
||||
|
||||
// ── Display ───────────────────────────────────────────────────
|
||||
@@ -204,17 +281,18 @@ function isStaleReduced12Match(
|
||||
|
||||
export interface CounterpartyTemplateMatch {
|
||||
template: CategorizationTemplate
|
||||
matchMethod: 'exact_alias' | 'exact_normalized' | 'fuzzy'
|
||||
matchMethod: 'exact_alias' | 'exact_normalized' | 'token_subset' | 'fuzzy'
|
||||
confidence: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a counterparty template matching a transaction.
|
||||
*
|
||||
* Three-tier matching (delegated to batch version with single-element array):
|
||||
* Four-tier matching (delegated to batch version with single-element array):
|
||||
* 1. Exact alias match
|
||||
* 2. Exact normalized name match
|
||||
* 3. Fuzzy Levenshtein: distance ≤2 for short names, ≤3 for long names
|
||||
* 3. Token subset: every distinctive template token appears in the descriptor
|
||||
* 4. Fuzzy Levenshtein: distance ≤2 for short names, ≤3 for long names
|
||||
*/
|
||||
export async function findCounterpartyTemplate(
|
||||
supabase: SupabaseClient,
|
||||
@@ -289,7 +367,49 @@ export async function findCounterpartyTemplatesBatch(
|
||||
continue
|
||||
}
|
||||
|
||||
// 3. Fuzzy Levenshtein match
|
||||
// 3. Token-subset match. Bridges the gaps exact and Levenshtein tiers
|
||||
// cannot: a template learned from manual bookings ("claude") whose token
|
||||
// appears inside a card descriptor ("ANTHROPIC* CLAUDE SUB SAN
|
||||
// FRANCISCO"), and a card-core descriptor ("anthropic") contained in a
|
||||
// template learned from the full splintered string before card-core
|
||||
// normalization existed. These differ by whole words, not typos.
|
||||
const txTokens = counterpartyTokenSet(rawName)
|
||||
const coreDistinct = distinctiveTokens(normalized.split(' '))
|
||||
let tokenBest: CategorizationTemplate | null = null
|
||||
let tokenBestShared = 0
|
||||
for (const tmpl of templates) {
|
||||
const tmplTokens = tmpl.counterparty_name.split(' ')
|
||||
const tmplDistinct = distinctiveTokens(tmplTokens)
|
||||
const templateInTx =
|
||||
tmplDistinct.length > 0 &&
|
||||
tmplDistinct.every((t) => txTokens.has(t)) &&
|
||||
(tmplDistinct.length >= 2 || tmpl.occurrence_count >= MIN_SINGLE_TOKEN_OCCURRENCES)
|
||||
const coreInTemplate =
|
||||
coreDistinct.length > 0 &&
|
||||
coreDistinct.every((t) => tmplTokens.includes(t)) &&
|
||||
(coreDistinct.length >= 2 || tmpl.occurrence_count >= MIN_SINGLE_TOKEN_OCCURRENCES)
|
||||
if (!templateInTx && !coreInTemplate) continue
|
||||
const shared = templateInTx ? tmplDistinct.length : coreDistinct.length
|
||||
if (
|
||||
shared > tokenBestShared ||
|
||||
(shared === tokenBestShared &&
|
||||
tokenBest !== null &&
|
||||
tmpl.occurrence_count > tokenBest.occurrence_count)
|
||||
) {
|
||||
tokenBestShared = shared
|
||||
tokenBest = tmpl
|
||||
}
|
||||
}
|
||||
if (tokenBest) {
|
||||
result.set(tx.id, {
|
||||
template: tokenBest,
|
||||
matchMethod: 'token_subset',
|
||||
confidence: Math.round(Number(tokenBest.confidence) * 0.85 * 100) / 100,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
// 4. Fuzzy Levenshtein match
|
||||
let bestMatch: CategorizationTemplate | null = null
|
||||
let bestDistance = Infinity
|
||||
for (const tmpl of templates) {
|
||||
|
||||
@@ -46,6 +46,40 @@ describe('buildMerchantHistory / merchantHistoryFor', () => {
|
||||
expect(merchantHistoryFor(map, 'Unknown Vendor')).toEqual({})
|
||||
expect(merchantHistoryFor(map, null)).toEqual({})
|
||||
})
|
||||
|
||||
it('falls back to the description when merchant_name is null (card purchases)', () => {
|
||||
// Bank feeds only carry counterparty names for transfers; card purchases
|
||||
// arrive with merchant_name null and the merchant buried in a descriptor
|
||||
// whose tail (product, city) changes between charges. All of these are
|
||||
// one counterparty: the reported Anthropic no-signal bug.
|
||||
const map = buildMerchantHistory([
|
||||
{ merchant_name: null, description: 'ANTHROPIC* CLAUDE SUB SAN FRANCISCO', category: 'expense_software' },
|
||||
{ merchant_name: null, description: 'ANTHROPIC*CLAUDE SUB +14155551234', category: 'expense_software' },
|
||||
{ merchant_name: 'Anthropic', description: 'irrelevant when merchant_name set', category: 'expense_software' },
|
||||
])
|
||||
expect(merchantHistoryFor(map, null, 'ANTHROPIC* CLAUDE SUB LONDON')).toEqual({
|
||||
expense_software: 3,
|
||||
})
|
||||
expect(merchantHistoryFor(map, 'Anthropic')).toEqual({ expense_software: 3 })
|
||||
})
|
||||
|
||||
it('anchors on original_description so user renames do not sever history', () => {
|
||||
// description is a mutable working title; a user renaming the row to
|
||||
// "Software" must not detach it from the raw bank descriptor identity.
|
||||
const map = buildMerchantHistory([
|
||||
{
|
||||
merchant_name: null,
|
||||
description: 'Software',
|
||||
original_description: 'ANTHROPIC* CLAUDE SUB SAN FRANCISCO',
|
||||
category: 'expense_software',
|
||||
},
|
||||
])
|
||||
expect(merchantHistoryFor(map, null, 'ANTHROPIC*CLAUDE SUB +14155551234')).toEqual({
|
||||
expense_software: 1,
|
||||
})
|
||||
// Renamed title itself is NOT a key when the raw descriptor exists.
|
||||
expect(merchantHistoryFor(map, null, 'Software')).toEqual({})
|
||||
})
|
||||
})
|
||||
|
||||
describe('getSuggestedCategories: counterparty history', () => {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { suggestCategory } from '@/lib/tax/expense-warnings'
|
||||
import { getExpenseAccountForCategory } from '@/lib/bookkeeping/category-mapping'
|
||||
import { normalizeCounterpartyName } from '@/lib/bookkeeping/counterparty-templates'
|
||||
import { findMatchingTemplates, getTemplateById, type TemplateMatch } from '@/lib/bookkeeping/booking-templates'
|
||||
import type { Transaction, TransactionCategory, EntityType, MappingRule, LinePatternEntry } from '@/types'
|
||||
|
||||
@@ -39,16 +40,39 @@ const CATEGORY_LABELS: Record<string, string> = {
|
||||
*/
|
||||
export type MerchantHistoryMap = Map<string, Record<string, number>>
|
||||
|
||||
function normalizeMerchantKey(name: string | null | undefined): string {
|
||||
return (name ?? '').toLowerCase().trim()
|
||||
/**
|
||||
* History keys share the counterparty-template normalization so card
|
||||
* descriptors ("ANTHROPIC* CLAUDE SUB SAN FRANCISCO") and clean merchant
|
||||
* names ("Anthropic") aggregate under one key. merchant_name is null on card
|
||||
* purchases (bank feeds only carry counterparty names for transfers), so the
|
||||
* descriptor is the fallback identity: without it, card merchants have no
|
||||
* history at all and every recurring foreign SaaS line reads as no-signal.
|
||||
* Callers pass `original_description ?? description`: the raw bank descriptor
|
||||
* is the stable anchor, `description` is a user-editable working title that
|
||||
* would sever the link on rename.
|
||||
*/
|
||||
function normalizeMerchantKey(
|
||||
merchantName: string | null | undefined,
|
||||
descriptor?: string | null,
|
||||
): string {
|
||||
const raw = (merchantName ?? '').trim() || (descriptor ?? '').trim()
|
||||
return raw ? normalizeCounterpartyName(raw) : ''
|
||||
}
|
||||
|
||||
export function buildMerchantHistory(
|
||||
rows: Array<{ merchant_name: string | null; category: string | null }>,
|
||||
rows: Array<{
|
||||
merchant_name: string | null
|
||||
description?: string | null
|
||||
original_description?: string | null
|
||||
category: string | null
|
||||
}>,
|
||||
): MerchantHistoryMap {
|
||||
const map: MerchantHistoryMap = new Map()
|
||||
for (const row of rows) {
|
||||
const key = normalizeMerchantKey(row.merchant_name)
|
||||
const key = normalizeMerchantKey(
|
||||
row.merchant_name,
|
||||
row.original_description ?? row.description,
|
||||
)
|
||||
if (!key || !row.category) continue
|
||||
const bucket = map.get(key) ?? {}
|
||||
bucket[row.category] = (bucket[row.category] || 0) + 1
|
||||
@@ -60,8 +84,9 @@ export function buildMerchantHistory(
|
||||
export function merchantHistoryFor(
|
||||
map: MerchantHistoryMap,
|
||||
merchantName: string | null | undefined,
|
||||
descriptor?: string | null,
|
||||
): Record<string, number> {
|
||||
const key = normalizeMerchantKey(merchantName)
|
||||
const key = normalizeMerchantKey(merchantName, descriptor)
|
||||
return key ? (map.get(key) ?? {}) : {}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
-- Card-descriptor awareness for normalize_counterparty_key().
|
||||
--
|
||||
-- Card-network descriptors embed '*' between merchant identity and a
|
||||
-- per-charge tail (product, order ref, city): "ANTHROPIC* CLAUDE SUB SAN
|
||||
-- FRANCISCO" is the merchant "ANTHROPIC", not five words. The tail varies
|
||||
-- between charges, so keeping it splinters every recurring foreign SaaS
|
||||
-- subscription into a new counterparty key each month; counterparty matching
|
||||
-- and ledger usage stats then report no signal for merchants with a dozen
|
||||
-- prior bookings. Processor descriptors ("PAYPAL *SPOTIFY", "SQ *BLUE
|
||||
-- BOTTLE") put the real merchant AFTER the star instead, keyed off a fixed
|
||||
-- processor prefix list.
|
||||
--
|
||||
-- This is the SQL mirror of extractCardDescriptorCore() inside
|
||||
-- normalizeCounterpartyName() (lib/bookkeeping/counterparty-templates.ts).
|
||||
-- Keep the two in sync: categorization_templates.counterparty_name is written
|
||||
-- through the TS side, and lib/agent-context/ledger-context.ts joins RPC rows
|
||||
-- to templates on this key. Parity is asserted by
|
||||
-- tests/pg/ledger-usage-stats-rpc.pg.test.ts.
|
||||
--
|
||||
-- CREATE OR REPLACE, no signature change: get_ledger_usage_stats() and the
|
||||
-- deep-context RPCs pick the new body up unchanged.
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.normalize_counterparty_key(raw text)
|
||||
RETURNS text
|
||||
LANGUAGE plpgsql
|
||||
IMMUTABLE
|
||||
PARALLEL SAFE
|
||||
AS $$
|
||||
DECLARE
|
||||
cleaned text;
|
||||
tokens text[];
|
||||
last_tok text;
|
||||
star_pos int;
|
||||
head text;
|
||||
tail text;
|
||||
head_key text;
|
||||
months constant text[] := ARRAY[
|
||||
'jan','feb','mar','apr','maj','may','jun','jul','aug','sep','sept',
|
||||
'okt','oct','nov','dec',
|
||||
'januari','februari','mars','april','juni','juli','augusti',
|
||||
'september','oktober','november','december'
|
||||
];
|
||||
-- Mirror of PROCESSOR_STAR_PREFIXES (counterparty-templates.ts).
|
||||
processors constant text[] := ARRAY[
|
||||
'paypal','klarna','izettle','zettle','iz','sq','sp','sumup',
|
||||
'google','stripe','payu','mollie'
|
||||
];
|
||||
BEGIN
|
||||
IF raw IS NULL THEN
|
||||
RETURN '';
|
||||
END IF;
|
||||
|
||||
-- stripBankNoise(): payment-rail prefixes, dates, invoice refs,
|
||||
-- trailing digit runs.
|
||||
cleaned := regexp_replace(raw, '^(BANKGIRO|SWISH|KORTKÖP|KORT[[:space:]]*KÖP|PG|BG|AUTOGIRO|PLUSGIRO)[[:space:]]*', '', 'i');
|
||||
cleaned := regexp_replace(cleaned, '\y\d{2,4}[-/]?\d{2}[-/]?\d{2}\y', '', 'g');
|
||||
cleaned := regexp_replace(cleaned, '\y[F#]?\d{4,}\S*', '', 'gi');
|
||||
cleaned := regexp_replace(cleaned, '\yINV-?\d+', '', 'gi');
|
||||
cleaned := regexp_replace(cleaned, '[[:space:]]+\d{4,}[[:space:]]*$', '', 'g');
|
||||
cleaned := btrim(cleaned);
|
||||
|
||||
-- extractCardDescriptorCore(): keep the merchant segment of a card
|
||||
-- descriptor. Head unless the head is a processor prefix; then tail.
|
||||
star_pos := position('*' in cleaned);
|
||||
IF star_pos > 0 THEN
|
||||
head := btrim(substr(cleaned, 1, star_pos - 1));
|
||||
tail := btrim(substr(cleaned, star_pos + 1));
|
||||
head_key := regexp_replace(lower(head), '[^a-z0-9åäöé]', '', 'g');
|
||||
IF head_key = ANY(processors) AND tail <> '' THEN
|
||||
cleaned := tail;
|
||||
ELSIF length(head_key) >= 3 THEN
|
||||
cleaned := head;
|
||||
ELSIF tail <> '' THEN
|
||||
cleaned := tail;
|
||||
ELSE
|
||||
cleaned := head;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
-- stripTrailingNoiseTokens(): pop trailing month names and 1-2 letter
|
||||
-- all-caps initials (checked against ORIGINAL casing, before lowering);
|
||||
-- always keep at least one token.
|
||||
tokens := regexp_split_to_array(cleaned, '[[:space:]]+');
|
||||
WHILE coalesce(array_length(tokens, 1), 0) > 1 LOOP
|
||||
last_tok := tokens[array_length(tokens, 1)];
|
||||
IF lower(last_tok) = ANY(months) OR last_tok ~ '^[A-ZÅÄÖ]{1,2}$' THEN
|
||||
tokens := tokens[1:array_length(tokens, 1) - 1];
|
||||
ELSE
|
||||
EXIT;
|
||||
END IF;
|
||||
END LOOP;
|
||||
cleaned := array_to_string(tokens, ' ');
|
||||
|
||||
-- normalizeMerchantName(): lowercase, strip special chars (keep Swedish
|
||||
-- letters), drop legal-form suffixes, collapse whitespace.
|
||||
cleaned := lower(cleaned);
|
||||
cleaned := regexp_replace(cleaned, '[^a-z0-9_[:space:]åäöé]', '', 'g');
|
||||
cleaned := regexp_replace(cleaned, '\y(ab|hb|kb|ek|för|stiftelse)\y', '', 'g');
|
||||
cleaned := regexp_replace(cleaned, '[[:space:]]+', ' ', 'g');
|
||||
RETURN btrim(cleaned);
|
||||
END;
|
||||
$$;
|
||||
@@ -223,6 +223,14 @@ describe('normalize_counterparty_key', () => {
|
||||
expect(await normalize('Ngrok Mars')).toBe('ngrok')
|
||||
// Invoice references.
|
||||
expect(await normalize('Acme INV-123')).toBe('acme')
|
||||
// Card-network descriptors: merchant segment before the star, per-charge
|
||||
// product/ref/city tail dropped (the Anthropic splinter bug). Processor
|
||||
// prefixes keep the merchant AFTER the star instead.
|
||||
expect(await normalize('ANTHROPIC* CLAUDE SUB SAN FRANCISCO')).toBe('anthropic')
|
||||
expect(await normalize('ANTHROPIC*CLAUDE SUB +14155551234')).toBe('anthropic')
|
||||
expect(await normalize('PAYPAL *SPOTIFY')).toBe('spotify')
|
||||
expect(await normalize('SQ *BLUE BOTTLE COFFEE')).toBe('blue bottle coffee')
|
||||
expect(await normalize('AMZN MKTP SE*A12B34CD5')).toBe('amzn mktp')
|
||||
// Never strips to empty: keeps the last token.
|
||||
expect(await normalize('SEB')).toBe('seb')
|
||||
// NULL-safe.
|
||||
|
||||
Reference in New Issue
Block a user