diff --git a/DECISIONS.md b/DECISIONS.md index e0e8c95c..aa66dcea 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -270,3 +270,4 @@ One line per decision: `[YYYY-MM-DD] : `. Appended by agents and [2026-07-21] Fortnox/SIE atomic import timeout fixed by function-scoped statement_timeout (290s) on import_sie_journal_entries, not by chunking the RPC: chunking would reintroduce the partial-import states the atomic RPC exists to eliminate (20260712150000), and the 20260629160100 delete-path precedent already uses the same bound; pg-real ratchet pins the config on all three SIE RPCs because CREATE OR REPLACE silently drops ALTER FUNCTION settings. [2026-07-21] Keep live annual-report narrative editable after a version is locked: immutable version snapshots preserve signing and filing content, while edits must remain possible to create a corrected superseding version. [2026-07-21] Restrict annual-report signature evidence transitions to the server service role and structured opaque references: browser RLS may manage only unbound pending roster rows, so route validation cannot be bypassed and evidence references cannot carry free-text personal data. +[2026-07-21] Card-descriptor normalization keys on the pre-star merchant segment (post-star for processor prefixes) plus a token_subset match tier, instead of the deferred AI descriptor normalization (data_quality_master Appendix B): deterministic, mirrors into normalize_counterparty_key() so ledger-context template joins stay exact, and fixes the reported Anthropic no-signal case with no new infrastructure. Merchant history now falls back to description because card purchases never carry merchant_name. diff --git a/app/api/transactions/suggest-categories/route.ts b/app/api/transactions/suggest-categories/route.ts index c3d28261..1bc5427c 100644 --- a/app/api/transactions/suggest-categories/route.ts +++ b/app/api/transactions/suggest-categories/route.ts @@ -44,11 +44,12 @@ export const POST = withRouteContext( // (global frequency padding produced identical low-confidence spreads). const { data: historicalTxns } = await supabase .from('transactions') - .select('category, merchant_name') + .select('category, merchant_name, description, original_description') .eq('company_id', companyId) .not('is_business', 'is', null) .neq('category', 'uncategorized') .neq('category', 'private') + .order('date', { ascending: false }) .limit(200) const merchantHistory = buildMerchantHistory(historicalTxns ?? []) @@ -72,7 +73,11 @@ export const POST = withRouteContext( suggestions[tx.id] = getSuggestedCategories( tx as Transaction, mappingRules || [], - merchantHistoryFor(merchantHistory, (tx as Transaction).merchant_name) + merchantHistoryFor( + merchantHistory, + (tx as Transaction).merchant_name, + (tx as Transaction).original_description ?? (tx as Transaction).description, + ) ) template_suggestions[tx.id] = await getSuggestedTemplates(tx as Transaction, entityType, mappingRules || undefined) } diff --git a/extensions/general/mcp-server/server.ts b/extensions/general/mcp-server/server.ts index 2aeb512e..68f6737f 100644 --- a/extensions/general/mcp-server/server.ts +++ b/extensions/general/mcp-server/server.ts @@ -5152,11 +5152,12 @@ export const tools: McpTool[] = [ // ~0.5 four-way spread agents reported as pure noise (P2-1). const { data: historicalTxns } = await supabase .from('transactions') - .select('category, merchant_name') + .select('category, merchant_name, description, original_description') .eq('company_id', companyId) .not('is_business', 'is', null) .neq('category', 'uncategorized') .neq('category', 'private') + .order('date', { ascending: false }) .limit(200) const merchantHistory = buildMerchantHistory(historicalTxns ?? []) @@ -5173,7 +5174,11 @@ export const tools: McpTool[] = [ for (const tx of transactions) { suggestions[tx.id] = getSuggestedCategories( tx as Transaction, mappingRules ?? [], - merchantHistoryFor(merchantHistory, (tx as Transaction).merchant_name) + merchantHistoryFor( + merchantHistory, + (tx as Transaction).merchant_name, + (tx as Transaction).original_description ?? (tx as Transaction).description, + ) ) const cpMatch = counterpartyMatches.get(tx.id) diff --git a/lib/bookkeeping/__tests__/counterparty-templates.test.ts b/lib/bookkeeping/__tests__/counterparty-templates.test.ts index 03314208..47196159 100644 --- a/lib/bookkeeping/__tests__/counterparty-templates.test.ts +++ b/lib/bookkeeping/__tests__/counterparty-templates.test.ts @@ -90,6 +90,21 @@ describe('counterparty-templates', () => { expect(normalizeCounterpartyName('NORDEA SEB')).toBe('nordea seb') // SEB is 3 chars: kept expect(normalizeCounterpartyName('KLARNA')).toBe('klarna') // single token kept }) + + it('reduces card-network descriptors to the merchant segment', () => { + // The Anthropic splinter bug: the per-charge tail after '*' (product, + // ref, city) changes between charges, so keeping it makes every + // recurring foreign SaaS subscription a new counterparty each month. + expect(normalizeCounterpartyName('ANTHROPIC* CLAUDE SUB SAN FRANCISCO')).toBe('anthropic') + expect(normalizeCounterpartyName('ANTHROPIC*CLAUDE SUB +14155551234')).toBe('anthropic') + expect(normalizeCounterpartyName('AMZN MKTP SE*A12B34CD5')).toBe('amzn mktp') + }) + + it('keeps the merchant AFTER the star for processor descriptors', () => { + expect(normalizeCounterpartyName('PAYPAL *SPOTIFY')).toBe('spotify') + expect(normalizeCounterpartyName('SQ *BLUE BOTTLE COFFEE')).toBe('blue bottle coffee') + expect(normalizeCounterpartyName('KLARNA*BOOZT FASHION')).toBe('boozt fashion') + }) }) // ── Confidence ───────────────────────────────────────────── @@ -219,6 +234,132 @@ describe('counterparty-templates', () => { expect(result).toBeNull() }) + + it('token-subset matches a template token buried in a card descriptor', async () => { + // The reported Anthropic case: a template learned from manual bookings + // ("Claude Dec" -> "claude") must match the bank's card descriptor even + // though the strings differ by whole words, not typos. + const template = makeCategorizationTemplate({ + counterparty_name: 'claude', + confidence: 0.8, + counterparty_aliases: ['claude dec'], + }) + const { supabase, enqueue } = createQueuedMockSupabase() + enqueue({ data: [template] }) + + const tx = makeTransaction({ + merchant_name: null, + description: 'ANTHROPIC* CLAUDE SUB SAN FRANCISCO', + }) + const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx) + + expect(result).not.toBeNull() + expect(result!.matchMethod).toBe('token_subset') + expect(result!.confidence).toBeCloseTo(0.68, 2) // 0.8 * 0.85 + }) + + it('token-subset matches a card-core descriptor against a legacy splintered template', async () => { + // Templates learned before card-core normalization stored the full + // descriptor; the new normalized core must still find them. + const template = makeCategorizationTemplate({ + counterparty_name: 'anthropic claude sub san francisco', + confidence: 0.7, + counterparty_aliases: [], + }) + const { supabase, enqueue } = createQueuedMockSupabase() + enqueue({ data: [template] }) + + const tx = makeTransaction({ + merchant_name: null, + description: 'ANTHROPIC*CLAUDE SUB LONDON', + }) + const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx) + + expect(result).not.toBeNull() + expect(result!.matchMethod).toBe('token_subset') + }) + + it('suppresses single-token matches without booking history behind them', async () => { + // A template named after a common first name (one prior booking) must + // not vacuum up an unrelated SWISH transfer that shares the name. + const template = makeCategorizationTemplate({ + counterparty_name: 'anders', + confidence: 0.5, + occurrence_count: 1, + counterparty_aliases: [], + }) + const { supabase, enqueue } = createQueuedMockSupabase() + enqueue({ data: [template] }) + + const tx = makeTransaction({ merchant_name: null, description: 'SWISH ANDERS JOHANSSON' }) + const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx) + + expect(result).toBeNull() + }) + + it('multi-token agreement matches without an occurrence floor', async () => { + const template = makeCategorizationTemplate({ + counterparty_name: 'blue bottle', + confidence: 0.5, + occurrence_count: 1, + counterparty_aliases: [], + }) + const { supabase, enqueue } = createQueuedMockSupabase() + enqueue({ data: [template] }) + + const tx = makeTransaction({ + merchant_name: null, + description: 'SQ *BLUE BOTTLE COFFEE OAKLAND', + }) + const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx) + + expect(result).not.toBeNull() + expect(result!.matchMethod).toBe('token_subset') + }) + + it('never token-matches on generic tokens alone', async () => { + // A template whose name is all generic/geo words must not vacuum up + // every descriptor mentioning them. + const template = makeCategorizationTemplate({ + counterparty_name: 'sverige', + confidence: 0.9, + counterparty_aliases: [], + }) + const { supabase, enqueue } = createQueuedMockSupabase() + enqueue({ data: [template] }) + + const tx = makeTransaction({ merchant_name: 'IKEA SVERIGE STOCKHOLM' }) + const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx) + + expect(result).toBeNull() + }) + + it('prefers exact normalized over token subset', async () => { + const exact = makeCategorizationTemplate({ + id: 'tmpl-exact', + counterparty_name: 'anthropic', + confidence: 0.8, + counterparty_aliases: [], + }) + const tokenish = makeCategorizationTemplate({ + id: 'tmpl-token', + counterparty_name: 'claude', + confidence: 0.8, + counterparty_aliases: [], + }) + const { supabase, enqueue } = createQueuedMockSupabase() + enqueue({ data: [tokenish, exact] }) + + const tx = makeTransaction({ + merchant_name: null, + description: 'ANTHROPIC* CLAUDE SUB SAN FRANCISCO', + }) + const result = await findCounterpartyTemplate(supabase as never, 'user-1', tx) + + expect(result).not.toBeNull() + expect(result!.matchMethod).toBe('exact_normalized') + expect(result!.template.id).toBe('tmpl-exact') + }) }) // ── Build MappingResult ──────────────────────────────────── diff --git a/lib/bookkeeping/counterparty-templates.ts b/lib/bookkeeping/counterparty-templates.ts index f2ec797c..7a06c187 100644 --- a/lib/bookkeeping/counterparty-templates.ts +++ b/lib/bookkeeping/counterparty-templates.ts @@ -64,15 +64,9 @@ function stripTrailingNoiseTokens(s: string): string { return tokens.join(' ') } -/** - * Normalize a transaction description to a canonical counterparty name. - * - * Strips bank transfer prefixes, trailing dates, invoice references, trailing - * digit sequences, and trailing period/initials tokens, then delegates to - * normalizeMerchantName() for Swedish company suffix removal and lowercasing. - */ -export function normalizeCounterpartyName(raw: string): string { - const cleaned = raw +/** Shared first stage: bank-feed noise that is never merchant identity. */ +function stripBankNoise(raw: string): string { + return raw // Strip common bank transfer prefixes .replace(/^(BANKGIRO|SWISH|KORTKÖP|KORT\s*KÖP|PG|BG|AUTOGIRO|PLUSGIRO)\s*/i, '') // Strip dates (20240615, 2024-06-15, 24-06-15) @@ -83,12 +77,95 @@ export function normalizeCounterpartyName(raw: string): string { // Strip trailing sequences of 4+ digits (card numbers, transaction refs) .replace(/\s+\d{4,}\s*$/g, '') .trim() +} + +/** + * Payment processors whose card descriptors put the real merchant AFTER the + * star ("PAYPAL *SPOTIFY", "SQ *BLUE BOTTLE"). For every other star + * descriptor the merchant is the segment before it. + */ +const PROCESSOR_STAR_PREFIXES = new Set([ + 'paypal', 'klarna', 'izettle', 'zettle', 'iz', 'sq', 'sp', 'sumup', + 'google', 'stripe', 'payu', 'mollie', +]) + +/** + * Reduce a card-network descriptor to its merchant segment. Card descriptors + * embed '*' between merchant identity and a per-charge tail (product, order + * ref, city): "ANTHROPIC* CLAUDE SUB SAN FRANCISCO" is the merchant + * "ANTHROPIC", not five words. The tail varies between charges, so keeping it + * splinters every recurring foreign SaaS subscription into a new counterparty + * each month. No '*' → returned unchanged. + * + * Mirrored in SQL by normalize_counterparty_key(); keep the two in sync + * (tests/pg/ledger-usage-stats-rpc.pg.test.ts asserts parity). + */ +function extractCardDescriptorCore(cleaned: string): string { + const starIdx = cleaned.indexOf('*') + if (starIdx === -1) return cleaned + const head = cleaned.slice(0, starIdx).trim() + const tail = cleaned.slice(starIdx + 1).trim() + const headKey = head.toLowerCase().replace(/[^a-z0-9åäöé]/g, '') + if (PROCESSOR_STAR_PREFIXES.has(headKey) && tail) return tail + if (headKey.length >= 3) return head + return tail || head +} + +/** + * Normalize a transaction description to a canonical counterparty name. + * + * Strips bank transfer prefixes, trailing dates, invoice references, trailing + * digit sequences, card-descriptor tails, and trailing period/initials tokens, + * then delegates to normalizeMerchantName() for Swedish company suffix removal + * and lowercasing. + */ +export function normalizeCounterpartyName(raw: string): string { + const cleaned = extractCardDescriptorCore(stripBankNoise(raw)) // Drop trailing month/initials tokens before merchant-name normalization so // "ngrok JW" and "Ngrok Mars" collapse to the same canonical "ngrok". return normalizeMerchantName(stripTrailingNoiseTokens(cleaned)) } +/** + * Full normalized token set of a descriptor WITHOUT the card-core reduction: + * the product segment of a card descriptor ("CLAUDE SUB") often carries the + * very token an existing template is named by ("claude", learned from manual + * bookings of the same subscription). Feeds the token_subset match tier only; + * canonical identity stays normalizeCounterpartyName(). + */ +export function counterpartyTokenSet(raw: string): Set { + const full = normalizeMerchantName(stripBankNoise(raw)) + return new Set(full.split(' ').filter(Boolean)) +} + +/** + * Tokens too generic to identify a merchant on their own: month labels, + * commerce noise, and geo words that ride along on bank descriptors. + */ +const GENERIC_TOKENS = new Set([ + ...TRAILING_MONTH_TOKENS, + 'subscription', 'subscr', 'abonnemang', 'betalning', 'payment', 'purchase', + 'online', 'store', 'shop', 'butik', 'faktura', 'invoice', + 'sweden', 'sverige', 'stockholm', 'göteborg', 'goteborg', 'malmö', 'malmo', +]) + +/** Distinctive = long and specific enough to identify a merchant. */ +function distinctiveTokens(tokens: string[]): string[] { + return tokens.filter( + (t) => t.length >= 4 && !GENERIC_TOKENS.has(t) && !/^\d+$/.test(t) + ) +} + +/** + * A single shared token is thin evidence: require the template to be backed + * by real booking history before trusting it, so a template named after a + * common word or first name (template "anders", occurrence 1) cannot vacuum + * up unrelated transfers ("SWISH ANDERS JOHANSSON"). Multi-token agreement + * is specific enough on its own. + */ +const MIN_SINGLE_TOKEN_OCCURRENCES = 3 + // ── Confidence ───────────────────────────────────────────────── // ── Display ─────────────────────────────────────────────────── @@ -204,17 +281,18 @@ function isStaleReduced12Match( export interface CounterpartyTemplateMatch { template: CategorizationTemplate - matchMethod: 'exact_alias' | 'exact_normalized' | 'fuzzy' + matchMethod: 'exact_alias' | 'exact_normalized' | 'token_subset' | 'fuzzy' confidence: number } /** * Find a counterparty template matching a transaction. * - * Three-tier matching (delegated to batch version with single-element array): + * Four-tier matching (delegated to batch version with single-element array): * 1. Exact alias match * 2. Exact normalized name match - * 3. Fuzzy Levenshtein: distance ≤2 for short names, ≤3 for long names + * 3. Token subset: every distinctive template token appears in the descriptor + * 4. Fuzzy Levenshtein: distance ≤2 for short names, ≤3 for long names */ export async function findCounterpartyTemplate( supabase: SupabaseClient, @@ -289,7 +367,49 @@ export async function findCounterpartyTemplatesBatch( continue } - // 3. Fuzzy Levenshtein match + // 3. Token-subset match. Bridges the gaps exact and Levenshtein tiers + // cannot: a template learned from manual bookings ("claude") whose token + // appears inside a card descriptor ("ANTHROPIC* CLAUDE SUB SAN + // FRANCISCO"), and a card-core descriptor ("anthropic") contained in a + // template learned from the full splintered string before card-core + // normalization existed. These differ by whole words, not typos. + const txTokens = counterpartyTokenSet(rawName) + const coreDistinct = distinctiveTokens(normalized.split(' ')) + let tokenBest: CategorizationTemplate | null = null + let tokenBestShared = 0 + for (const tmpl of templates) { + const tmplTokens = tmpl.counterparty_name.split(' ') + const tmplDistinct = distinctiveTokens(tmplTokens) + const templateInTx = + tmplDistinct.length > 0 && + tmplDistinct.every((t) => txTokens.has(t)) && + (tmplDistinct.length >= 2 || tmpl.occurrence_count >= MIN_SINGLE_TOKEN_OCCURRENCES) + const coreInTemplate = + coreDistinct.length > 0 && + coreDistinct.every((t) => tmplTokens.includes(t)) && + (coreDistinct.length >= 2 || tmpl.occurrence_count >= MIN_SINGLE_TOKEN_OCCURRENCES) + if (!templateInTx && !coreInTemplate) continue + const shared = templateInTx ? tmplDistinct.length : coreDistinct.length + if ( + shared > tokenBestShared || + (shared === tokenBestShared && + tokenBest !== null && + tmpl.occurrence_count > tokenBest.occurrence_count) + ) { + tokenBestShared = shared + tokenBest = tmpl + } + } + if (tokenBest) { + result.set(tx.id, { + template: tokenBest, + matchMethod: 'token_subset', + confidence: Math.round(Number(tokenBest.confidence) * 0.85 * 100) / 100, + }) + continue + } + + // 4. Fuzzy Levenshtein match let bestMatch: CategorizationTemplate | null = null let bestDistance = Infinity for (const tmpl of templates) { diff --git a/lib/transactions/__tests__/category-suggestions.test.ts b/lib/transactions/__tests__/category-suggestions.test.ts index 68ddddef..dd57dd8a 100644 --- a/lib/transactions/__tests__/category-suggestions.test.ts +++ b/lib/transactions/__tests__/category-suggestions.test.ts @@ -46,6 +46,40 @@ describe('buildMerchantHistory / merchantHistoryFor', () => { expect(merchantHistoryFor(map, 'Unknown Vendor')).toEqual({}) expect(merchantHistoryFor(map, null)).toEqual({}) }) + + it('falls back to the description when merchant_name is null (card purchases)', () => { + // Bank feeds only carry counterparty names for transfers; card purchases + // arrive with merchant_name null and the merchant buried in a descriptor + // whose tail (product, city) changes between charges. All of these are + // one counterparty: the reported Anthropic no-signal bug. + const map = buildMerchantHistory([ + { merchant_name: null, description: 'ANTHROPIC* CLAUDE SUB SAN FRANCISCO', category: 'expense_software' }, + { merchant_name: null, description: 'ANTHROPIC*CLAUDE SUB +14155551234', category: 'expense_software' }, + { merchant_name: 'Anthropic', description: 'irrelevant when merchant_name set', category: 'expense_software' }, + ]) + expect(merchantHistoryFor(map, null, 'ANTHROPIC* CLAUDE SUB LONDON')).toEqual({ + expense_software: 3, + }) + expect(merchantHistoryFor(map, 'Anthropic')).toEqual({ expense_software: 3 }) + }) + + it('anchors on original_description so user renames do not sever history', () => { + // description is a mutable working title; a user renaming the row to + // "Software" must not detach it from the raw bank descriptor identity. + const map = buildMerchantHistory([ + { + merchant_name: null, + description: 'Software', + original_description: 'ANTHROPIC* CLAUDE SUB SAN FRANCISCO', + category: 'expense_software', + }, + ]) + expect(merchantHistoryFor(map, null, 'ANTHROPIC*CLAUDE SUB +14155551234')).toEqual({ + expense_software: 1, + }) + // Renamed title itself is NOT a key when the raw descriptor exists. + expect(merchantHistoryFor(map, null, 'Software')).toEqual({}) + }) }) describe('getSuggestedCategories: counterparty history', () => { diff --git a/lib/transactions/category-suggestions.ts b/lib/transactions/category-suggestions.ts index 6b1bd2dd..5a04f5c3 100644 --- a/lib/transactions/category-suggestions.ts +++ b/lib/transactions/category-suggestions.ts @@ -1,5 +1,6 @@ import { suggestCategory } from '@/lib/tax/expense-warnings' import { getExpenseAccountForCategory } from '@/lib/bookkeeping/category-mapping' +import { normalizeCounterpartyName } from '@/lib/bookkeeping/counterparty-templates' import { findMatchingTemplates, getTemplateById, type TemplateMatch } from '@/lib/bookkeeping/booking-templates' import type { Transaction, TransactionCategory, EntityType, MappingRule, LinePatternEntry } from '@/types' @@ -39,16 +40,39 @@ const CATEGORY_LABELS: Record = { */ export type MerchantHistoryMap = Map> -function normalizeMerchantKey(name: string | null | undefined): string { - return (name ?? '').toLowerCase().trim() +/** + * History keys share the counterparty-template normalization so card + * descriptors ("ANTHROPIC* CLAUDE SUB SAN FRANCISCO") and clean merchant + * names ("Anthropic") aggregate under one key. merchant_name is null on card + * purchases (bank feeds only carry counterparty names for transfers), so the + * descriptor is the fallback identity: without it, card merchants have no + * history at all and every recurring foreign SaaS line reads as no-signal. + * Callers pass `original_description ?? description`: the raw bank descriptor + * is the stable anchor, `description` is a user-editable working title that + * would sever the link on rename. + */ +function normalizeMerchantKey( + merchantName: string | null | undefined, + descriptor?: string | null, +): string { + const raw = (merchantName ?? '').trim() || (descriptor ?? '').trim() + return raw ? normalizeCounterpartyName(raw) : '' } export function buildMerchantHistory( - rows: Array<{ merchant_name: string | null; category: string | null }>, + rows: Array<{ + merchant_name: string | null + description?: string | null + original_description?: string | null + category: string | null + }>, ): MerchantHistoryMap { const map: MerchantHistoryMap = new Map() for (const row of rows) { - const key = normalizeMerchantKey(row.merchant_name) + const key = normalizeMerchantKey( + row.merchant_name, + row.original_description ?? row.description, + ) if (!key || !row.category) continue const bucket = map.get(key) ?? {} bucket[row.category] = (bucket[row.category] || 0) + 1 @@ -60,8 +84,9 @@ export function buildMerchantHistory( export function merchantHistoryFor( map: MerchantHistoryMap, merchantName: string | null | undefined, + descriptor?: string | null, ): Record { - const key = normalizeMerchantKey(merchantName) + const key = normalizeMerchantKey(merchantName, descriptor) return key ? (map.get(key) ?? {}) : {} } diff --git a/supabase/migrations/20260722070000_normalize_counterparty_key_card_descriptors.sql b/supabase/migrations/20260722070000_normalize_counterparty_key_card_descriptors.sql new file mode 100644 index 00000000..5c53b5f1 --- /dev/null +++ b/supabase/migrations/20260722070000_normalize_counterparty_key_card_descriptors.sql @@ -0,0 +1,102 @@ +-- Card-descriptor awareness for normalize_counterparty_key(). +-- +-- Card-network descriptors embed '*' between merchant identity and a +-- per-charge tail (product, order ref, city): "ANTHROPIC* CLAUDE SUB SAN +-- FRANCISCO" is the merchant "ANTHROPIC", not five words. The tail varies +-- between charges, so keeping it splinters every recurring foreign SaaS +-- subscription into a new counterparty key each month; counterparty matching +-- and ledger usage stats then report no signal for merchants with a dozen +-- prior bookings. Processor descriptors ("PAYPAL *SPOTIFY", "SQ *BLUE +-- BOTTLE") put the real merchant AFTER the star instead, keyed off a fixed +-- processor prefix list. +-- +-- This is the SQL mirror of extractCardDescriptorCore() inside +-- normalizeCounterpartyName() (lib/bookkeeping/counterparty-templates.ts). +-- Keep the two in sync: categorization_templates.counterparty_name is written +-- through the TS side, and lib/agent-context/ledger-context.ts joins RPC rows +-- to templates on this key. Parity is asserted by +-- tests/pg/ledger-usage-stats-rpc.pg.test.ts. +-- +-- CREATE OR REPLACE, no signature change: get_ledger_usage_stats() and the +-- deep-context RPCs pick the new body up unchanged. + +CREATE OR REPLACE FUNCTION public.normalize_counterparty_key(raw text) +RETURNS text +LANGUAGE plpgsql +IMMUTABLE +PARALLEL SAFE +AS $$ +DECLARE + cleaned text; + tokens text[]; + last_tok text; + star_pos int; + head text; + tail text; + head_key text; + months constant text[] := ARRAY[ + 'jan','feb','mar','apr','maj','may','jun','jul','aug','sep','sept', + 'okt','oct','nov','dec', + 'januari','februari','mars','april','juni','juli','augusti', + 'september','oktober','november','december' + ]; + -- Mirror of PROCESSOR_STAR_PREFIXES (counterparty-templates.ts). + processors constant text[] := ARRAY[ + 'paypal','klarna','izettle','zettle','iz','sq','sp','sumup', + 'google','stripe','payu','mollie' + ]; +BEGIN + IF raw IS NULL THEN + RETURN ''; + END IF; + + -- stripBankNoise(): payment-rail prefixes, dates, invoice refs, + -- trailing digit runs. + cleaned := regexp_replace(raw, '^(BANKGIRO|SWISH|KORTKÖP|KORT[[:space:]]*KÖP|PG|BG|AUTOGIRO|PLUSGIRO)[[:space:]]*', '', 'i'); + cleaned := regexp_replace(cleaned, '\y\d{2,4}[-/]?\d{2}[-/]?\d{2}\y', '', 'g'); + cleaned := regexp_replace(cleaned, '\y[F#]?\d{4,}\S*', '', 'gi'); + cleaned := regexp_replace(cleaned, '\yINV-?\d+', '', 'gi'); + cleaned := regexp_replace(cleaned, '[[:space:]]+\d{4,}[[:space:]]*$', '', 'g'); + cleaned := btrim(cleaned); + + -- extractCardDescriptorCore(): keep the merchant segment of a card + -- descriptor. Head unless the head is a processor prefix; then tail. + star_pos := position('*' in cleaned); + IF star_pos > 0 THEN + head := btrim(substr(cleaned, 1, star_pos - 1)); + tail := btrim(substr(cleaned, star_pos + 1)); + head_key := regexp_replace(lower(head), '[^a-z0-9åäöé]', '', 'g'); + IF head_key = ANY(processors) AND tail <> '' THEN + cleaned := tail; + ELSIF length(head_key) >= 3 THEN + cleaned := head; + ELSIF tail <> '' THEN + cleaned := tail; + ELSE + cleaned := head; + END IF; + END IF; + + -- stripTrailingNoiseTokens(): pop trailing month names and 1-2 letter + -- all-caps initials (checked against ORIGINAL casing, before lowering); + -- always keep at least one token. + tokens := regexp_split_to_array(cleaned, '[[:space:]]+'); + WHILE coalesce(array_length(tokens, 1), 0) > 1 LOOP + last_tok := tokens[array_length(tokens, 1)]; + IF lower(last_tok) = ANY(months) OR last_tok ~ '^[A-ZÅÄÖ]{1,2}$' THEN + tokens := tokens[1:array_length(tokens, 1) - 1]; + ELSE + EXIT; + END IF; + END LOOP; + cleaned := array_to_string(tokens, ' '); + + -- normalizeMerchantName(): lowercase, strip special chars (keep Swedish + -- letters), drop legal-form suffixes, collapse whitespace. + cleaned := lower(cleaned); + cleaned := regexp_replace(cleaned, '[^a-z0-9_[:space:]åäöé]', '', 'g'); + cleaned := regexp_replace(cleaned, '\y(ab|hb|kb|ek|för|stiftelse)\y', '', 'g'); + cleaned := regexp_replace(cleaned, '[[:space:]]+', ' ', 'g'); + RETURN btrim(cleaned); +END; +$$; diff --git a/tests/pg/ledger-usage-stats-rpc.pg.test.ts b/tests/pg/ledger-usage-stats-rpc.pg.test.ts index 87af96fa..3d377825 100644 --- a/tests/pg/ledger-usage-stats-rpc.pg.test.ts +++ b/tests/pg/ledger-usage-stats-rpc.pg.test.ts @@ -223,6 +223,14 @@ describe('normalize_counterparty_key', () => { expect(await normalize('Ngrok Mars')).toBe('ngrok') // Invoice references. expect(await normalize('Acme INV-123')).toBe('acme') + // Card-network descriptors: merchant segment before the star, per-charge + // product/ref/city tail dropped (the Anthropic splinter bug). Processor + // prefixes keep the merchant AFTER the star instead. + expect(await normalize('ANTHROPIC* CLAUDE SUB SAN FRANCISCO')).toBe('anthropic') + expect(await normalize('ANTHROPIC*CLAUDE SUB +14155551234')).toBe('anthropic') + expect(await normalize('PAYPAL *SPOTIFY')).toBe('spotify') + expect(await normalize('SQ *BLUE BOTTLE COFFEE')).toBe('blue bottle coffee') + expect(await normalize('AMZN MKTP SE*A12B34CD5')).toBe('amzn mktp') // Never strips to empty: keeps the last token. expect(await normalize('SEB')).toBe('seb') // NULL-safe.