fix(categorization): connect card descriptors to counterparty history (#1095)

* fix(categorization): connect card descriptors to counterparty history suggest_categories returned no signal for recurring card merchants (reported: Anthropic booked to 5420 fourteen times, zero suggestions). Three compounding causes, all fixed: - normalizeCounterpartyName() now reduces card-network descriptors to their merchant segment ("ANTHROPIC* CLAUDE SUB SAN FRANCISCO" -> "anthropic"; "PAYPAL *SPOTIFY" -> "spotify"), so monthly per-charge tails stop splintering one merchant into unmatchable variants. SQL mirror normalize_counterparty_key() updated in lockstep (migration 20260721140000), keeping the ledger-context template join exact. - New token_subset match tier bridges templates learned from manual bookings ("Claude Dec" -> "claude") to bank descriptors containing the token, and card-core descriptors to legacy splintered templates. Guarded by a distinctive-token filter so generic/geo words never match on their own. - Merchant history falls back to description when merchant_name is null: card purchases never carry merchant_name, so the history path was structurally blind to exactly the transactions that need it. History keys now share the counterparty-template normalization and the 200-row window is ordered by recency. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(categorization): guard single-token matches, anchor history on original_description Review follow-ups (CodeRabbit on #1095): - token_subset tier: a single shared distinctive token now also requires occurrence_count >= 3 on the template, so a template named after a common word or first name (one prior booking) cannot vacuum up unrelated transfers ("SWISH ANDERS JOHANSSON"). Multi-token agreement stays unrestricted; the Claude/Anthropic case (14 bookings) is unaffected. - merchant history keys on original_description ?? description: the raw bank descriptor is immutable while description is a user-editable working title, so renaming a transaction no longer severs its history link for future recurring charges. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * chore(migrations): re-timestamp card-descriptor migration after prod moved past it Prod applied 20260721144311 (#1101) through 20260721201747 (#1104) while this PR was open; 20260721140000 would sort before them and risk being skipped by out-of-order auto-apply at merge. Not yet applied to prod, so renaming is safe; the preview branch re-applies idempotently (CREATE OR REPLACE). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-07-22 15:58:45 +02:00
parent 3920c893f4
commit 4a0b524fbb
9 changed files with 463 additions and 22 deletions
@@ -90,6 +90,21 @@ describe('counterparty-templates', () => {
      expect(normalizeCounterpartyName('NORDEA SEB')).toBe('nordea seb') // SEB is 3 chars: kept
      expect(normalizeCounterpartyName('KLARNA')).toBe('klarna')         // single token kept
    })
+
+    it('reduces card-network descriptors to the merchant segment', () => {
+      // The Anthropic splinter bug: the per-charge tail after '*' (product,
+      // ref, city) changes between charges, so keeping it makes every
+      // recurring foreign SaaS subscription a new counterparty each month.
+      expect(normalizeCounterpartyName('ANTHROPIC* CLAUDE SUB SAN FRANCISCO')).toBe('anthropic')
+      expect(normalizeCounterpartyName('ANTHROPIC*CLAUDE SUB +14155551234')).toBe('anthropic')
+      expect(normalizeCounterpartyName('AMZN MKTP SE*A12B34CD5')).toBe('amzn mktp')
+    })
+
+    it('keeps the merchant AFTER the star for processor descriptors', () => {
+      expect(normalizeCounterpartyName('PAYPAL *SPOTIFY')).toBe('spotify')
+      expect(normalizeCounterpartyName('SQ *BLUE BOTTLE COFFEE')).toBe('blue bottle coffee')
+      expect(normalizeCounterpartyName('KLARNA*BOOZT FASHION')).toBe('boozt fashion')
+    })
  })

  // ── Confidence ─────────────────────────────────────────────
@@ -219,6 +234,132 @@ describe('counterparty-templates', () => {

      expect(result).toBeNull()
    })
+
+    it('token-subset matches a template token buried in a card descriptor', async () => {
+      // The reported Anthropic case: a template learned from manual bookings
+      // ("Claude Dec" -> "claude") must match the bank's card descriptor even
+      // though the strings differ by whole words, not typos.
+      const template = makeCategorizationTemplate({
+        counterparty_name: 'claude',
+        confidence: 0.8,
+        counterparty_aliases: ['claude dec'],
+      })
+      const { supabase, enqueue } = createQueuedMockSupabase()
+      enqueue({ data: [template] })
+
+      const tx = makeTransaction({
+        merchant_name: null,
+        description: 'ANTHROPIC* CLAUDE SUB SAN FRANCISCO',
+      })
+      const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx)
+
+      expect(result).not.toBeNull()
+      expect(result!.matchMethod).toBe('token_subset')
+      expect(result!.confidence).toBeCloseTo(0.68, 2) // 0.8 * 0.85
+    })
+
+    it('token-subset matches a card-core descriptor against a legacy splintered template', async () => {
+      // Templates learned before card-core normalization stored the full
+      // descriptor; the new normalized core must still find them.
+      const template = makeCategorizationTemplate({
+        counterparty_name: 'anthropic claude sub san francisco',
+        confidence: 0.7,
+        counterparty_aliases: [],
+      })
+      const { supabase, enqueue } = createQueuedMockSupabase()
+      enqueue({ data: [template] })
+
+      const tx = makeTransaction({
+        merchant_name: null,
+        description: 'ANTHROPIC*CLAUDE SUB LONDON',
+      })
+      const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx)
+
+      expect(result).not.toBeNull()
+      expect(result!.matchMethod).toBe('token_subset')
+    })
+
+    it('suppresses single-token matches without booking history behind them', async () => {
+      // A template named after a common first name (one prior booking) must
+      // not vacuum up an unrelated SWISH transfer that shares the name.
+      const template = makeCategorizationTemplate({
+        counterparty_name: 'anders',
+        confidence: 0.5,
+        occurrence_count: 1,
+        counterparty_aliases: [],
+      })
+      const { supabase, enqueue } = createQueuedMockSupabase()
+      enqueue({ data: [template] })
+
+      const tx = makeTransaction({ merchant_name: null, description: 'SWISH ANDERS JOHANSSON' })
+      const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx)
+
+      expect(result).toBeNull()
+    })
+
+    it('multi-token agreement matches without an occurrence floor', async () => {
+      const template = makeCategorizationTemplate({
+        counterparty_name: 'blue bottle',
+        confidence: 0.5,
+        occurrence_count: 1,
+        counterparty_aliases: [],
+      })
+      const { supabase, enqueue } = createQueuedMockSupabase()
+      enqueue({ data: [template] })
+
+      const tx = makeTransaction({
+        merchant_name: null,
+        description: 'SQ *BLUE BOTTLE COFFEE OAKLAND',
+      })
+      const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx)
+
+      expect(result).not.toBeNull()
+      expect(result!.matchMethod).toBe('token_subset')
+    })
+
+    it('never token-matches on generic tokens alone', async () => {
+      // A template whose name is all generic/geo words must not vacuum up
+      // every descriptor mentioning them.
+      const template = makeCategorizationTemplate({
+        counterparty_name: 'sverige',
+        confidence: 0.9,
+        counterparty_aliases: [],
+      })
+      const { supabase, enqueue } = createQueuedMockSupabase()
+      enqueue({ data: [template] })
+
+      const tx = makeTransaction({ merchant_name: 'IKEA SVERIGE STOCKHOLM' })
+      const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx)
+
+      expect(result).toBeNull()
+    })
+
+    it('prefers exact normalized over token subset', async () => {
+      const exact = makeCategorizationTemplate({
+        id: 'tmpl-exact',
+        counterparty_name: 'anthropic',
+        confidence: 0.8,
+        counterparty_aliases: [],
+      })
+      const tokenish = makeCategorizationTemplate({
+        id: 'tmpl-token',
+        counterparty_name: 'claude',
+        confidence: 0.8,
+        counterparty_aliases: [],
+      })
+      const { supabase, enqueue } = createQueuedMockSupabase()
+      enqueue({ data: [tokenish, exact] })
+
+      const tx = makeTransaction({
+        merchant_name: null,
+        description: 'ANTHROPIC* CLAUDE SUB SAN FRANCISCO',
+      })
+      const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx)
+
+      expect(result).not.toBeNull()
+      expect(result!.matchMethod).toBe('exact_normalized')
+      expect(result!.template.id).toBe('tmpl-exact')
+    })
  })

  // ── Build MappingResult ────────────────────────────────────
@@ -64,15 +64,9 @@ function stripTrailingNoiseTokens(s: string): string {
  return tokens.join(' ')
 }

-/**
- * Normalize a transaction description to a canonical counterparty name.
- *
- * Strips bank transfer prefixes, trailing dates, invoice references, trailing
- * digit sequences, and trailing period/initials tokens, then delegates to
- * normalizeMerchantName() for Swedish company suffix removal and lowercasing.
- */
-export function normalizeCounterpartyName(raw: string): string {
-  const cleaned = raw
+/** Shared first stage: bank-feed noise that is never merchant identity. */
+function stripBankNoise(raw: string): string {
+  return raw
    // Strip common bank transfer prefixes
    .replace(/^(BANKGIRO|SWISH|KORTKÖP|KORT\s*KÖP|PG|BG|AUTOGIRO|PLUSGIRO)\s*/i, '')
    // Strip dates (20240615, 2024-06-15, 24-06-15)
@@ -83,12 +77,95 @@ export function normalizeCounterpartyName(raw: string): string {
    // Strip trailing sequences of 4+ digits (card numbers, transaction refs)
    .replace(/\s+\d{4,}\s*$/g, '')
    .trim()
+}
+
+/**
+ * Payment processors whose card descriptors put the real merchant AFTER the
+ * star ("PAYPAL *SPOTIFY", "SQ *BLUE BOTTLE"). For every other star
+ * descriptor the merchant is the segment before it.
+ */
+const PROCESSOR_STAR_PREFIXES = new Set([
+  'paypal', 'klarna', 'izettle', 'zettle', 'iz', 'sq', 'sp', 'sumup',
+  'google', 'stripe', 'payu', 'mollie',
+])
+
+/**
+ * Reduce a card-network descriptor to its merchant segment. Card descriptors
+ * embed '*' between merchant identity and a per-charge tail (product, order
+ * ref, city): "ANTHROPIC* CLAUDE SUB SAN FRANCISCO" is the merchant
+ * "ANTHROPIC", not five words. The tail varies between charges, so keeping it
+ * splinters every recurring foreign SaaS subscription into a new counterparty
+ * each month. No '*' → returned unchanged.
+ *
+ * Mirrored in SQL by normalize_counterparty_key(); keep the two in sync
+ * (tests/pg/ledger-usage-stats-rpc.pg.test.ts asserts parity).
+ */
+function extractCardDescriptorCore(cleaned: string): string {
+  const starIdx = cleaned.indexOf('*')
+  if (starIdx === -1) return cleaned
+  const head = cleaned.slice(0, starIdx).trim()
+  const tail = cleaned.slice(starIdx + 1).trim()
+  const headKey = head.toLowerCase().replace(/[^a-z0-9åäöé]/g, '')
+  if (PROCESSOR_STAR_PREFIXES.has(headKey) && tail) return tail
+  if (headKey.length >= 3) return head
+  return tail || head
+}
+
+/**
+ * Normalize a transaction description to a canonical counterparty name.
+ *
+ * Strips bank transfer prefixes, trailing dates, invoice references, trailing
+ * digit sequences, card-descriptor tails, and trailing period/initials tokens,
+ * then delegates to normalizeMerchantName() for Swedish company suffix removal
+ * and lowercasing.
+ */
+export function normalizeCounterpartyName(raw: string): string {
+  const cleaned = extractCardDescriptorCore(stripBankNoise(raw))

  // Drop trailing month/initials tokens before merchant-name normalization so
  // "ngrok JW" and "Ngrok Mars" collapse to the same canonical "ngrok".
  return normalizeMerchantName(stripTrailingNoiseTokens(cleaned))
 }

+/**
+ * Full normalized token set of a descriptor WITHOUT the card-core reduction:
+ * the product segment of a card descriptor ("CLAUDE SUB") often carries the
+ * very token an existing template is named by ("claude", learned from manual
+ * bookings of the same subscription). Feeds the token_subset match tier only;
+ * canonical identity stays normalizeCounterpartyName().
+ */
+export function counterpartyTokenSet(raw: string): Set<string> {
+  const full = normalizeMerchantName(stripBankNoise(raw))
+  return new Set(full.split(' ').filter(Boolean))
+}
+
+/**
+ * Tokens too generic to identify a merchant on their own: month labels,
+ * commerce noise, and geo words that ride along on bank descriptors.
+ */
+const GENERIC_TOKENS = new Set([
+  ...TRAILING_MONTH_TOKENS,
+  'subscription', 'subscr', 'abonnemang', 'betalning', 'payment', 'purchase',
+  'online', 'store', 'shop', 'butik', 'faktura', 'invoice',
+  'sweden', 'sverige', 'stockholm', 'göteborg', 'goteborg', 'malmö', 'malmo',
+])
+
+/** Distinctive = long and specific enough to identify a merchant. */
+function distinctiveTokens(tokens: string[]): string[] {
+  return tokens.filter(
+    (t) => t.length >= 4 && !GENERIC_TOKENS.has(t) && !/^\d+$/.test(t)
+  )
+}
+
+/**
+ * A single shared token is thin evidence: require the template to be backed
+ * by real booking history before trusting it, so a template named after a
+ * common word or first name (template "anders", occurrence 1) cannot vacuum
+ * up unrelated transfers ("SWISH ANDERS JOHANSSON"). Multi-token agreement
+ * is specific enough on its own.
+ */
+const MIN_SINGLE_TOKEN_OCCURRENCES = 3
+
 // ── Confidence ─────────────────────────────────────────────────

 // ── Display ───────────────────────────────────────────────────
@@ -204,17 +281,18 @@ function isStaleReduced12Match(

 export interface CounterpartyTemplateMatch {
  template: CategorizationTemplate
-  matchMethod: 'exact_alias' | 'exact_normalized' | 'fuzzy'
+  matchMethod: 'exact_alias' | 'exact_normalized' | 'token_subset' | 'fuzzy'
  confidence: number
 }

 /**
 * Find a counterparty template matching a transaction.
 *
- * Three-tier matching (delegated to batch version with single-element array):
+ * Four-tier matching (delegated to batch version with single-element array):
 * 1. Exact alias match
 * 2. Exact normalized name match
- * 3. Fuzzy Levenshtein: distance ≤2 for short names, ≤3 for long names
+ * 3. Token subset: every distinctive template token appears in the descriptor
+ * 4. Fuzzy Levenshtein: distance ≤2 for short names, ≤3 for long names
 */
 export async function findCounterpartyTemplate(
  supabase: SupabaseClient,
@@ -289,7 +367,49 @@ export async function findCounterpartyTemplatesBatch(
      continue
    }

-    // 3. Fuzzy Levenshtein match
+    // 3. Token-subset match. Bridges the gaps exact and Levenshtein tiers
+    // cannot: a template learned from manual bookings ("claude") whose token
+    // appears inside a card descriptor ("ANTHROPIC* CLAUDE SUB SAN
+    // FRANCISCO"), and a card-core descriptor ("anthropic") contained in a
+    // template learned from the full splintered string before card-core
+    // normalization existed. These differ by whole words, not typos.
+    const txTokens = counterpartyTokenSet(rawName)
+    const coreDistinct = distinctiveTokens(normalized.split(' '))
+    let tokenBest: CategorizationTemplate | null = null
+    let tokenBestShared = 0
+    for (const tmpl of templates) {
+      const tmplTokens = tmpl.counterparty_name.split(' ')
+      const tmplDistinct = distinctiveTokens(tmplTokens)
+      const templateInTx =
+        tmplDistinct.length > 0 &&
+        tmplDistinct.every((t) => txTokens.has(t)) &&
+        (tmplDistinct.length >= 2 || tmpl.occurrence_count >= MIN_SINGLE_TOKEN_OCCURRENCES)
+      const coreInTemplate =
+        coreDistinct.length > 0 &&
+        coreDistinct.every((t) => tmplTokens.includes(t)) &&
+        (coreDistinct.length >= 2 || tmpl.occurrence_count >= MIN_SINGLE_TOKEN_OCCURRENCES)
+      if (!templateInTx && !coreInTemplate) continue
+      const shared = templateInTx ? tmplDistinct.length : coreDistinct.length
+      if (
+        shared > tokenBestShared ||
+        (shared === tokenBestShared &&
+          tokenBest !== null &&
+          tmpl.occurrence_count > tokenBest.occurrence_count)
+      ) {
+        tokenBestShared = shared
+        tokenBest = tmpl
+      }
+    }
+    if (tokenBest) {
+      result.set(tx.id, {
+        template: tokenBest,
+        matchMethod: 'token_subset',
+        confidence: Math.round(Number(tokenBest.confidence) * 0.85 * 100) / 100,
+      })
+      continue
+    }
+
+    // 4. Fuzzy Levenshtein match
    let bestMatch: CategorizationTemplate | null = null
    let bestDistance = Infinity
    for (const tmpl of templates) {
@@ -46,6 +46,40 @@ describe('buildMerchantHistory / merchantHistoryFor', () => {
    expect(merchantHistoryFor(map, 'Unknown Vendor')).toEqual({})
    expect(merchantHistoryFor(map, null)).toEqual({})
  })
+
+  it('falls back to the description when merchant_name is null (card purchases)', () => {
+    // Bank feeds only carry counterparty names for transfers; card purchases
+    // arrive with merchant_name null and the merchant buried in a descriptor
+    // whose tail (product, city) changes between charges. All of these are
+    // one counterparty: the reported Anthropic no-signal bug.
+    const map = buildMerchantHistory([
+      { merchant_name: null, description: 'ANTHROPIC* CLAUDE SUB SAN FRANCISCO', category: 'expense_software' },
+      { merchant_name: null, description: 'ANTHROPIC*CLAUDE SUB +14155551234', category: 'expense_software' },
+      { merchant_name: 'Anthropic', description: 'irrelevant when merchant_name set', category: 'expense_software' },
+    ])
+    expect(merchantHistoryFor(map, null, 'ANTHROPIC* CLAUDE SUB LONDON')).toEqual({
+      expense_software: 3,
+    })
+    expect(merchantHistoryFor(map, 'Anthropic')).toEqual({ expense_software: 3 })
+  })
+
+  it('anchors on original_description so user renames do not sever history', () => {
+    // description is a mutable working title; a user renaming the row to
+    // "Software" must not detach it from the raw bank descriptor identity.
+    const map = buildMerchantHistory([
+      {
+        merchant_name: null,
+        description: 'Software',
+        original_description: 'ANTHROPIC* CLAUDE SUB SAN FRANCISCO',
+        category: 'expense_software',
+      },
+    ])
+    expect(merchantHistoryFor(map, null, 'ANTHROPIC*CLAUDE SUB +14155551234')).toEqual({
+      expense_software: 1,
+    })
+    // Renamed title itself is NOT a key when the raw descriptor exists.
+    expect(merchantHistoryFor(map, null, 'Software')).toEqual({})
+  })
 })

 describe('getSuggestedCategories: counterparty history', () => {
@@ -1,5 +1,6 @@
 import { suggestCategory } from '@/lib/tax/expense-warnings'
 import { getExpenseAccountForCategory } from '@/lib/bookkeeping/category-mapping'
+import { normalizeCounterpartyName } from '@/lib/bookkeeping/counterparty-templates'
 import { findMatchingTemplates, getTemplateById, type TemplateMatch } from '@/lib/bookkeeping/booking-templates'
 import type { Transaction, TransactionCategory, EntityType, MappingRule, LinePatternEntry } from '@/types'

@@ -39,16 +40,39 @@ const CATEGORY_LABELS: Record<string, string> = {
 */
 export type MerchantHistoryMap = Map<string, Record<string, number>>

-function normalizeMerchantKey(name: string | null | undefined): string {
-  return (name ?? '').toLowerCase().trim()
+/**
+ * History keys share the counterparty-template normalization so card
+ * descriptors ("ANTHROPIC* CLAUDE SUB SAN FRANCISCO") and clean merchant
+ * names ("Anthropic") aggregate under one key. merchant_name is null on card
+ * purchases (bank feeds only carry counterparty names for transfers), so the
+ * descriptor is the fallback identity: without it, card merchants have no
+ * history at all and every recurring foreign SaaS line reads as no-signal.
+ * Callers pass `original_description ?? description`: the raw bank descriptor
+ * is the stable anchor, `description` is a user-editable working title that
+ * would sever the link on rename.
+ */
+function normalizeMerchantKey(
+  merchantName: string | null | undefined,
+  descriptor?: string | null,
+): string {
+  const raw = (merchantName ?? '').trim() || (descriptor ?? '').trim()
+  return raw ? normalizeCounterpartyName(raw) : ''
 }

 export function buildMerchantHistory(
-  rows: Array<{ merchant_name: string | null; category: string | null }>,
+  rows: Array<{
+    merchant_name: string | null
+    description?: string | null
+    original_description?: string | null
+    category: string | null
+  }>,
 ): MerchantHistoryMap {
  const map: MerchantHistoryMap = new Map()
  for (const row of rows) {
-    const key = normalizeMerchantKey(row.merchant_name)
+    const key = normalizeMerchantKey(
+      row.merchant_name,
+      row.original_description ?? row.description,
+    )
    if (!key || !row.category) continue
    const bucket = map.get(key) ?? {}
    bucket[row.category] = (bucket[row.category] || 0) + 1
@@ -60,8 +84,9 @@ export function buildMerchantHistory(
 export function merchantHistoryFor(
  map: MerchantHistoryMap,
  merchantName: string | null | undefined,
+  descriptor?: string | null,
 ): Record<string, number> {
-  const key = normalizeMerchantKey(merchantName)
+  const key = normalizeMerchantKey(merchantName, descriptor)
  return key ? (map.get(key) ?? {}) : {}
 }