diff --git a/app/api/v1/companies/[companyId]/transactions/ingest/route.ts b/app/api/v1/companies/[companyId]/transactions/ingest/route.ts index d584e74b..2ac383ca 100644 --- a/app/api/v1/companies/[companyId]/transactions/ingest/route.ts +++ b/app/api/v1/companies/[companyId]/transactions/ingest/route.ts @@ -22,7 +22,7 @@ import { registerEndpoint } from '@/lib/api/v1/registry' import { withApiV1 } from '@/lib/api/v1/with-api-v1' import { v1ErrorResponse, v1ErrorResponseFromCode } from '@/lib/api/v1/errors' import { ingestTransactions } from '@/lib/transactions/ingest' -import { contentDedupKey } from '@/lib/transactions/external-id' +import { contentBucketKey, descriptionsBridge, normalizeImportedDescription } from '@/lib/transactions/external-id' import type { RawTransaction } from '@/types' const RawTx = z.object({ @@ -70,10 +70,10 @@ registerEndpoint({ 'Single ad-hoc transactions (use the dashboard). Documents/receipts (use the documents endpoint). Manually-created journal entries (Phase 4).', pitfalls: [ 'external_id is the primary dedup key — make it stable for the same physical transaction across reruns.', - 'Content-based dedup (date+amount) runs in addition: a CSV row that matches an already-booked transaction by date+amount is skipped even if external_id differs.', + 'Content-based dedup runs in addition: a row matching an already-booked transaction by date, amount AND description (prefix-containment, to survive PSD2 title enrichment) is skipped even if external_id differs.', 'raw_insert_only=true skips ALL post-insert pipeline steps (matching, categorization). Use for viewer-only imports.', 'Max 500 items per call. For larger imports, split into pages of 500.', - 'Dry-run runs both dedup checks (external_id AND content-based date+amount against booked rows), matching the live pipeline. Numbers should agree barring concurrent imports between preview and commit.', + 'Dry-run previews external_id + content dedup against BOOKED rows only; the live pipeline also dedups against unbooked bank-synced rows, so preview skips are a lower bound on the live skip count.', ], example: { request: { @@ -153,25 +153,52 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>( const { data: bookedInRange } = await ctx.supabase .from('transactions') - .select('date, amount, description') + .select('date, amount, original_description, description') .eq('company_id', ctx.companyId!) .not('journal_entry_id', 'is', null) .gte('date', dateFrom) .lte('date', dateTo) - // Build the content-dedup key with the SAME helper the live pipeline uses - // (lib/transactions/ingest.ts), so the preview's content-match decision - // matches the eventual ingest exactly: öre-normalized amount (handles a - // PostgREST numeric returned as a string) plus the description prefix. - const bookedKeys = new Set( - (bookedInRange ?? []).map((r) => { - const row = r as { date: string; amount: number | string; description: string | null } - return contentDedupKey(row.date, row.amount, row.description) - }), - ) + // Mirror the live pipeline's content-dedup bridge (lib/transactions/ingest.ts): + // bucket booked rows by (date, öre) and match by description prefix-containment + // (keyed off the immutable original_description), consumed with the SAME + // longest-match + counting semantics — so a batch of N copies against M booked + // twins previews M skips and N−M imports, not N. Scope note: like the live + // pipeline's booked map this previews ONLY booked rows; the unbooked + // enable_banking overlap check is not modelled here. + const bookedBuckets = new Map() + for (const r of bookedInRange ?? []) { + const row = r as { date: string; amount: number | string; original_description: string | null; description: string | null } + const key = contentBucketKey(row.date, row.amount) + const desc = normalizeImportedDescription(row.original_description ?? row.description).toLowerCase().trim() + const bucket = bookedBuckets.get(key) + if (bucket) bucket.push(desc) + else bookedBuckets.set(key, [desc]) + } + // Consume the longest bridging stored description (counting semantics) — the + // same logic as ingest.ts consumeBridgingTwin, on this preview's mutable copy. + const consumeBookedTwin = (date: string, amount: number, desc: string): boolean => { + const descs = bookedBuckets.get(contentBucketKey(date, amount)) + if (!descs || descs.length === 0) return false + let bestIdx = -1 + let bestLen = -1 + for (let i = 0; i < descs.length; i++) { + if (descriptionsBridge(desc, descs[i]) && descs[i].length > bestLen) { + bestIdx = i + bestLen = descs[i].length + } + } + if (bestIdx === -1) return false + descs.splice(bestIdx, 1) + return true + } const previewRows = body.transactions.map((tx) => { const extIdHit = knownExtIds.has(tx.external_id) - const contentHit = bookedKeys.has(contentDedupKey(tx.date, tx.amount, tx.description)) + // external_id precedence mirrors the live pipeline: a row caught by + // external_id must NOT consume a booked twin (so it stays available for a + // genuine content-only duplicate later in the batch). + const contentHit = + !extIdHit && consumeBookedTwin(tx.date, tx.amount, normalizeImportedDescription(tx.description)) const wouldSkip = extIdHit || contentHit const reason = extIdHit ? 'external_id_match' diff --git a/lib/transactions/__tests__/external-id.test.ts b/lib/transactions/__tests__/external-id.test.ts index 52b7cd89..fb3016f0 100644 --- a/lib/transactions/__tests__/external-id.test.ts +++ b/lib/transactions/__tests__/external-id.test.ts @@ -2,7 +2,8 @@ import { describe, it, expect } from 'vitest' import { amountToOre, buildStableExternalIds, - contentDedupKey, + contentBucketKey, + descriptionsBridge, normalizeImportedDescription, FALLBACK_DESCRIPTION, } from '../external-id' @@ -87,35 +88,67 @@ describe('buildStableExternalIds', () => { it('returns an empty array for an empty batch', () => { expect(buildStableExternalIds('eb', 'acc', [])).toEqual([]) }) + + // FORMAT-FREEZE guard. `external_id` is a STORED key: changing the template + // string orphans every prior row (its stored id stops matching the new scheme) + // and re-imports the lot on the next sync — the June 2026 fleet-wide incident. + // If you must change the format, you MUST ship a coordinated backfill of + // existing rows; updating this assertion without one is the bug. + it('FORMAT IS FROZEN — changing this template silently orphans every prior external_id', () => { + expect( + buildStableExternalIds('eb', 'SE0000000000000000000000', [ + { date: '2026-04-07', amount: -11231 }, + ]), + ).toEqual(['eb_SE0000000000000000000000_2026-04-07_-1123100_0']) + }) }) -describe('contentDedupKey', () => { +describe('contentBucketKey', () => { + it('keys off (date, öre) only — no description', () => { + expect(contentBucketKey('2024-06-15', -250)).toBe('2024-06-15|-25000') + }) + it('matches a JS number against a PostgREST numeric string for the same amount', () => { - // The core dedup-bridge fix: an incoming raw number and a DB-fetched string - // for the same amount + date + description must produce the SAME key. - const incoming = contentDedupKey('2024-06-15', -250, 'ICA Maxi Solna') - const stored = contentDedupKey('2024-06-15', '-250.00', 'ICA Maxi Solna') - expect(incoming).toBe(stored) + // A DB-fetched numeric string and a raw JS number for the same amount must + // land in the SAME bucket, otherwise the bridge silently misses. + expect(contentBucketKey('2024-06-15', -250)).toBe(contentBucketKey('2024-06-15', '-250.00')) }) - it('normalizes description (lowercase, trim, 24-char prefix)', () => { - expect(contentDedupKey('2024-06-15', -100, ' ICA Maxi Solna ')) - .toBe(contentDedupKey('2024-06-15', -100, 'ica maxi solna')) - // Differs only past the 24-char prefix → same key. - expect(contentDedupKey('2024-06-15', -100, 'Betalning till leverantör AAA')) - .toBe(contentDedupKey('2024-06-15', -100, 'Betalning till leverantör BBB')) + it('separates distinct amounts and dates into distinct buckets', () => { + expect(contentBucketKey('2024-06-15', -250)).not.toBe(contentBucketKey('2024-06-15', -100)) + expect(contentBucketKey('2024-06-15', -250)).not.toBe(contentBucketKey('2024-06-16', -250)) + }) +}) + +describe('descriptionsBridge', () => { + it('bridges prefix-preserving PSD2 enrichment (the June 2026 drift)', () => { + // The same transaction whose title grew between syncs must still bridge. + // (Synthetic stand-ins for the real prefix-preserving enrichment pattern.) + expect(descriptionsBridge('KAFFE', 'KAFFE BG 0000000000 Bg-bet. via internet')).toBe(true) + expect(descriptionsBridge('UTBETALNING Insättning', 'UTBETALNING')).toBe(true) + expect(descriptionsBridge('REF 000000 Europabetalning', 'REF 000000')).toBe(true) }) - it('keeps distinct transactions apart when description differs in the prefix', () => { - expect(contentDedupKey('2024-06-15', -250, 'ICA Maxi')) - .not.toBe(contentDedupKey('2024-06-15', -250, 'Coop Stockholm')) + it('is case- and whitespace-insensitive', () => { + expect(descriptionsBridge(' Mataffär Solna ', 'mataffär solna')).toBe(true) }) - it('treats a null/undefined description as an empty prefix', () => { - expect(contentDedupKey('2024-06-15', -100, null)) - .toBe(contentDedupKey('2024-06-15', -100, undefined)) - expect(contentDedupKey('2024-06-15', -100, null)) - .toBe(contentDedupKey('2024-06-15', -100, '')) + it('does NOT bridge genuinely distinct descriptions sharing a date+amount', () => { + // Distinct reference codes on same-day same-amount rows (e.g. verification + // micro-deposits) must NOT collapse — each is a real transaction. + expect(descriptionsBridge('REF-AAAA1111', 'REF-BBBB2222')).toBe(false) + // Same common stem, diverging tails — still distinct. + expect(descriptionsBridge('PMT.Ref AAA', 'PMT.Ref BBB')).toBe(false) + expect(descriptionsBridge('Coffee', 'Lunch')).toBe(false) + }) + + it('does not let a blank description wildcard-match a described row', () => { + // A blank carries no signal: it must not consume a described same-(date,öre) + // row. Live callers normalize blanks to FALLBACK_DESCRIPTION, so this is + // defense-in-depth. Only two blanks bridge each other (date+öre identity). + expect(descriptionsBridge('', 'anything')).toBe(false) + expect(descriptionsBridge('anything', null)).toBe(false) + expect(descriptionsBridge(undefined, '')).toBe(true) }) }) diff --git a/lib/transactions/__tests__/ingest.test.ts b/lib/transactions/__tests__/ingest.test.ts index 15c89071..45cda80d 100644 --- a/lib/transactions/__tests__/ingest.test.ts +++ b/lib/transactions/__tests__/ingest.test.ts @@ -390,6 +390,186 @@ describe('ingestTransactions', () => { expect(result.transaction_ids).toEqual(['tx-no-collision']) }) + // ----------------------------------------------------------------------- + // 2d. Description drift: PSD2 enrichment is prefix-preserving, so an + // enriched re-import ("TIC" → "TIC BG … via internet") still bridges + // the stored original via prefix-containment. This is the June 2026 + // incident: the external_id ALSO changed, so the bridge is the only net. + // ----------------------------------------------------------------------- + it('dedupes an enriched re-import whose description extends the stored original', async () => { + const { supabase, enqueue } = createQueueMockSupabase() + const raw = makeRaw({ + date: '2026-04-07', + amount: -11231, + description: 'KAFFE BG 0000000000 Bg-bet. via internet', // enriched + external_id: 'eb_SE00_2026-04-07_-1123100_0', // NEW-scheme id → external_id dedup misses + import_source: 'enable_banking', + }) + + enqueue({ data: [], error: null }) // booked map — none + // Unbooked enable_banking row carrying the SHORT original description. + enqueue({ + data: [{ date: '2026-04-07', amount: -11231, original_description: 'KAFFE', description: 'KAFFE' }], + error: null, + }) + enqueue({ data: [], error: null }) // supplier invoices + enqueue({ data: [], error: null }) // external_id dedup — different scheme, no match + + const result = await ingestTransactions(supabase as never, COMPANY_ID, USER_ID, [raw]) + + expect(result.duplicates).toBe(1) + expect(result.imported).toBe(0) + }) + + // ----------------------------------------------------------------------- + // 2e. Order-independence: a genuinely-new row whose description does NOT + // bridge an existing same-(date,amount) row is kept, and the re-import + // that DOES bridge is deduped — regardless of provider ordering. + // ----------------------------------------------------------------------- + it.each([ + ['new-first', ['Lunch', 'Coffee']], + ['dup-first', ['Coffee', 'Lunch']], + ])('keeps the distinct row and dedupes the bridging twin (%s)', async (_label, order) => { + const { supabase, enqueue } = createQueueMockSupabase() + const rows = order.map((desc, i) => + makeRaw({ + date: '2026-04-07', + amount: -250, + description: desc, + external_id: `csv_${desc}_${i}`, + import_source: 'csv_lunar', + }), + ) + const insertedDesc = 'Lunch' // the non-bridging "Lunch" is always the row that gets inserted + + enqueue({ data: [], error: null }) // booked map — none + // One unbooked enable_banking row "Coffee" — only the incoming "Coffee" bridges it. + enqueue({ + data: [{ date: '2026-04-07', amount: -250, original_description: 'Coffee', description: 'Coffee' }], + error: null, + }) + enqueue({ data: [], error: null }) // supplier invoices + enqueue({ data: [], error: null }) // external_id dedup — no match + enqueue({ + data: makeTransaction({ id: 'tx-lunch', description: insertedDesc, amount: -250 }), + error: null, + }) // insert for the non-bridging "Lunch" + mockEvaluateMappingRules.mockResolvedValue(makeMappingResult({ confidence: 0.5 })) + + const result = await ingestTransactions(supabase as never, COMPANY_ID, USER_ID, rows) + + expect(result.imported).toBe(1) // "Lunch" kept + expect(result.duplicates).toBe(1) // "Coffee" deduped + expect(result.transaction_ids).toEqual(['tx-lunch']) + }) + + // ----------------------------------------------------------------------- + // 2f. Counting semantics: N stored twins dedup exactly N incoming bridging + // rows; the surplus is inserted (never silently collapsed). + // ----------------------------------------------------------------------- + it('dedupes exactly as many incoming rows as there are stored twins (counting)', async () => { + const { supabase, enqueue } = createQueueMockSupabase() + const rows = [1, 2, 3].map((n) => + makeRaw({ + date: '2026-04-07', + amount: -100, + description: `ICA Kortköp ${n}`, + external_id: `csv_ica_${n}`, + import_source: 'csv_lunar', + }), + ) + + enqueue({ data: [], error: null }) // booked map — none + // Two stored unbooked "ICA" twins → only two of the three incoming dedup. + enqueue({ + data: [ + { date: '2026-04-07', amount: -100, original_description: 'ICA', description: 'ICA' }, + { date: '2026-04-07', amount: -100, original_description: 'ICA', description: 'ICA' }, + ], + error: null, + }) + enqueue({ data: [], error: null }) // supplier invoices + enqueue({ data: [], error: null }) // external_id dedup — no match + enqueue({ + data: makeTransaction({ id: 'tx-ica-surplus', description: 'ICA Kortköp 3', amount: -100 }), + error: null, + }) // insert for the surplus third row + mockEvaluateMappingRules.mockResolvedValue(makeMappingResult({ confidence: 0.5 })) + + const result = await ingestTransactions(supabase as never, COMPANY_ID, USER_ID, rows) + + expect(result.duplicates).toBe(2) + expect(result.imported).toBe(1) + expect(result.transaction_ids).toEqual(['tx-ica-surplus']) + }) + + // ----------------------------------------------------------------------- + // 2g. Cross-account guard: a transaction on one bank account must NOT + // deduplicate a genuinely-different one on ANOTHER account of the same + // company. The content bucket is company-wide (only external_id embeds + // the account), so the bridge also requires matching cash_account_id when + // both sides know it. + // ----------------------------------------------------------------------- + it('does not dedupe a bridging twin that settled on a different cash account', async () => { + const { supabase, enqueue } = createQueueMockSupabase() + const raw = makeRaw({ + date: '2026-04-07', + amount: -250, + description: 'Avgift', + external_id: 'eb_acctB_2026-04-07_-25000_0', + import_source: 'enable_banking', + }) + const inserted = makeTransaction({ id: 'tx-acctB', amount: -250 }) + + enqueue({ data: [], error: null }) // booked map — none + // Unbooked enable_banking twin, but it settled on a DIFFERENT account (A). + enqueue({ + data: [{ date: '2026-04-07', amount: -250, original_description: 'Avgift', description: 'Avgift', cash_account_id: 'acct-A' }], + error: null, + }) + enqueue({ data: [], error: null }) // supplier invoices + enqueue({ data: [], error: null }) // external_id dedup — no match + enqueue({ data: { id: 'acct-B' }, error: null }) // cash_accounts lookup → batch settled on account B + enqueue({ data: inserted, error: null }) // insert — not a duplicate + mockEvaluateMappingRules.mockResolvedValue(makeMappingResult({ confidence: 0.5 })) + + const result = await ingestTransactions(supabase as never, COMPANY_ID, USER_ID, [raw], { + settlementAccount: '1931', + }) + + expect(result.imported).toBe(1) + expect(result.duplicates).toBe(0) + expect(result.transaction_ids).toEqual(['tx-acctB']) + }) + + it('dedupes a bridging twin on the SAME cash account', async () => { + const { supabase, enqueue } = createQueueMockSupabase() + const raw = makeRaw({ + date: '2026-04-07', + amount: -250, + description: 'Avgift', + external_id: 'eb_acctA_2026-04-07_-25000_99', // different id → external_id dedup misses + import_source: 'enable_banking', + }) + + enqueue({ data: [], error: null }) // booked map — none + enqueue({ + data: [{ date: '2026-04-07', amount: -250, original_description: 'Avgift', description: 'Avgift', cash_account_id: 'acct-A' }], + error: null, + }) + enqueue({ data: [], error: null }) // supplier invoices + enqueue({ data: [], error: null }) // external_id dedup — no match + enqueue({ data: { id: 'acct-A' }, error: null }) // cash_accounts lookup → batch settled on account A (same) + // No insert — deduped. + + const result = await ingestTransactions(supabase as never, COMPANY_ID, USER_ID, [raw], { + settlementAccount: '1930', + }) + + expect(result.duplicates).toBe(1) + expect(result.imported).toBe(0) + }) + // ----------------------------------------------------------------------- // 3. Counts errors when insert fails // ----------------------------------------------------------------------- diff --git a/lib/transactions/external-id.ts b/lib/transactions/external-id.ts index ba3b96aa..f9a1222b 100644 --- a/lib/transactions/external-id.ts +++ b/lib/transactions/external-id.ts @@ -65,6 +65,14 @@ export function normalizeImportedDescription(raw: string | null | undefined): st * — where `n` is an occurrence index that disambiguates genuinely identical * transactions (same account, date and amount) within the batch. * + * ⚠️ THE FORMAT STRING IS A STORED KEY. It is persisted to + * `transactions.external_id` and dedup compares incoming ids against the stored + * ones byte-for-byte. Changing this template silently orphans every prior row + * (its stored id no longer matches the new scheme) and re-imports them all on + * the next sync — this is exactly what happened in the June 2026 fleet-wide + * incident. Any format change MUST ship a coordinated backfill of existing rows + * and is locked by a frozen-format test (see `external-id.test.ts`). + * * Properties this guarantees: * - **Re-sync dedupe**: the same set of transactions produces the same *set* * of ids regardless of the order the ASPSP returns them in, so a repeat sync @@ -77,13 +85,13 @@ export function normalizeImportedDescription(raw: string | null | undefined): st * This is the safeguard the bank-file importer already relies on via its * `rowIndex` component (see `lib/import/bank-file/parser.ts`). * - * Why description is NOT an input here (but IS in `contentDedupKey`): the + * Why description is NOT an input here (but IS in the content bridge): the * `external_id` must be a *stable unique key*, so it cannot depend on a field * that drifts — PSD2 enriches/reorders descriptions between a transaction's * pending and booked states. The occurrence index gives uniqueness without - * that fragility. `contentDedupKey` has the opposite job — it is a best-effort - * *bridge* that must avoid dropping real transactions — so it keeps the - * description (see that function for the trade-off). + * that fragility. The content bridge (`contentBucketKey` + `descriptionsBridge`) + * has the opposite job — it is a best-effort *bridge* that must avoid dropping + * real transactions — so it keeps the description (see those for the trade-off). * * @param prefix Source tag, e.g. `'eb'` for Enable Banking. * @param accountScope Stable per-account scope (prefer IBAN, fall back to the @@ -108,32 +116,51 @@ export function buildStableExternalIds( } /** - * Stable content-dedup key used to bridge transactions across `external_id` - * schemes and import sources (PSD2 ⇄ CSV, old id scheme ⇄ new id scheme). + * Bucket key for the content-dedup bridge: `{date}|{öre}` — deliberately NO + * description. Transactions that share a date and amount fall into the same + * bucket; `descriptionsBridge` then decides, per pair, whether two rows in that + * bucket are the same transaction. Splitting bucketing (date+öre) from matching + * (description) is what lets the bridge survive description drift while still + * keeping genuinely-distinct same-(date,amount) transactions apart. * - * Format: `{date}|{öre}|{normalized description prefix}`. - * - * This is a *best-effort* dedup signal, not a unique key. It is consumed with - * COUNTING semantics in the ingest pipeline (N existing matches consume N - * incoming), and its job is to skip re-imports WITHOUT ever dropping a real - * transaction. That asymmetry drives the two design choices here: - * - * - **öre via `amountToOre`** — a JS number (`-250`) and a PostgREST numeric - * string (`"-250.00"`) must collapse to the same key, otherwise dedup - * silently misses. - * - **description IS included** (unlike `external_id`) — two genuinely distinct - * transactions that merely share a date and amount (e.g. two SEK 250 card - * purchases) must NOT be collapsed into one, or a real transaction is lost. - * Including the description prefix biases toward keeping both. The cost is - * that if a description drifts between syncs the bridge can miss a true - * duplicate — an acceptable trade for an accounting ledger, where a visible, - * user-deletable duplicate is far safer than a silently dropped row. + * öre via `amountToOre` so a JS number (`-250`) and a PostgREST numeric string + * (`"-250.00"`) collapse to the same bucket, otherwise dedup silently misses. */ -export function contentDedupKey( - date: string, - amount: number | string, - description: string | null | undefined -): string { - const descPrefix = (description || '').toLowerCase().trim().slice(0, 24) - return `${date}|${amountToOre(amount)}|${descPrefix}` +export function contentBucketKey(date: string, amount: number | string): string { + return `${date}|${amountToOre(amount)}` +} + +/** + * Decide whether two normalized descriptions (same date+öre bucket) describe the + * same underlying transaction — the matching half of the content-dedup bridge. + * + * Returns true when either description is a prefix of the other. PSD2 enrichment + * is **prefix-preserving**: the same transaction's title grows between syncs + * ("TIC" → "TIC BG 0000005786439 Bg-bet. via internet", "UTBETALNING" → + * "UTBETALNING Insättning"), so prefix-containment bridges the two where a + * fixed-length prefix *equality* check (the pre-June-2026 scheme) missed and + * re-imported. A blank description carries no signal, so it never bridges a + * *described* row — otherwise an empty title would wildcard-match any + * same-(date,öre) transaction and could silently consume a real one; only two + * blanks bridge each other (date+öre identity). In practice every caller + * normalizes blanks to FALLBACK_DESCRIPTION upstream (see + * normalizeImportedDescription), so the blank path is defense-in-depth. + * Genuinely distinct descriptions ("Coffee" vs "Lunch", or two different + * reference codes that share a date and amount) are NOT prefixes of one another + * and never bridge, so two real same-(date,amount) transactions are kept apart. + * + * This is a *best-effort* signal, consumed with COUNTING semantics in the ingest + * pipeline (N existing matches consume N incoming): its job is to skip re-imports + * WITHOUT ever dropping a real transaction. The asymmetry favours keeping a + * visible, user-deletable duplicate over silently losing a row. + */ +export function descriptionsBridge( + a: string | null | undefined, + b: string | null | undefined +): boolean { + const x = (a ?? '').toLowerCase().trim() + const y = (b ?? '').toLowerCase().trim() + // A blank never wildcards a described row; only two blanks bridge each other. + if (x === '' || y === '') return x === y + return x.startsWith(y) || y.startsWith(x) } diff --git a/lib/transactions/ingest.ts b/lib/transactions/ingest.ts index dd187fcb..fa49c186 100644 --- a/lib/transactions/ingest.ts +++ b/lib/transactions/ingest.ts @@ -7,15 +7,32 @@ import { findSupplierInvoiceMatch } from '@/lib/invoices/supplier-invoice-matchi import { fetchExchangeRate } from '@/lib/currency/riksbanken' import { logMatchEvent } from '@/lib/invoices/match-log' import { fetchAllRows } from '@/lib/supabase/fetch-all' -import { contentDedupKey, normalizeImportedDescription } from '@/lib/transactions/external-id' +import { contentBucketKey, descriptionsBridge, normalizeImportedDescription } from '@/lib/transactions/external-id' import type { Transaction, RawTransaction, IngestResult, IngestOptions, SupplierInvoice, Currency, ExchangeRate } from '@/types' // Re-export types for backward compatibility export type { RawTransaction, IngestResult } from '@/types' +/** + * One existing row in a content-dedup bucket: its normalized/lowercased + * description plus the cash account it settled on (null for legacy rows that + * predate the cash_account_id backfill). `cashAccountId` is the cross-account + * guard — see `consumeBridgingTwin`. + */ +type BucketEntry = { desc: string; cashAccountId: string | null } + +/** + * Content-dedup bucket: a `{date}|{öre}` key mapped to the multiset of existing + * rows in that bucket. Matching is by `descriptionsBridge` (prefix-containment) + * gated by the account guard, consumed with COUNTING semantics — one entry is + * spliced out per deduped incoming row — so two genuinely-distinct + * same-(date,amount) transactions are never collapsed. + */ +type DescBucket = Map + interface ExistingTransactionMaps { /** Booked transactions (any source) — consumed by any incoming raw transaction. */ - booked: Map + booked: DescBucket /** * Unbooked enable_banking transactions — consumed by any incoming raw * transaction regardless of source. Catches two cases: PSD2 reconnect @@ -23,7 +40,22 @@ interface ExistingTransactionMaps { * CSV imports overlapping an active PSD2 sync (same Lunar/etc tx arriving * twice, once via PSD2 and once via file upload). */ - unbookedEnableBanking: Map + unbookedEnableBanking: DescBucket +} + +/** Push a row into its (date, öre) bucket, normalizing the description. */ +function addToBucket( + bucket: DescBucket, + date: string, + amount: number | string, + description: string, + cashAccountId: string | null, +): void { + const key = contentBucketKey(date, amount) + const entry: BucketEntry = { desc: description.toLowerCase().trim(), cashAccountId } + const entries = bucket.get(key) + if (entries) entries.push(entry) + else bucket.set(key, [entry]) } async function buildExistingTransactionMaps( @@ -31,8 +63,8 @@ async function buildExistingTransactionMaps( companyId: string, rawTransactions: RawTransaction[] ): Promise { - const booked = new Map() - const unbookedEnableBanking = new Map() + const booked: DescBucket = new Map() + const unbookedEnableBanking: DescBucket = new Map() if (rawTransactions.length === 0) return { booked, unbookedEnableBanking } const dates = rawTransactions.map((t) => t.date).sort() @@ -42,7 +74,7 @@ async function buildExistingTransactionMaps( try { const { data: bookedRows } = await supabase .from('transactions') - .select('date, amount, original_description, description') + .select('date, amount, original_description, description, cash_account_id') .eq('company_id', companyId) .not('journal_entry_id', 'is', null) .gte('date', dateFrom) @@ -54,12 +86,13 @@ async function buildExistingTransactionMaps( // description: a title edit must never make the dedup bridge miss a // genuine re-import. Falls back to description for rows predating the // original_description column. - const key = contentDedupKey( + addToBucket( + booked, tx.date, tx.amount, normalizeImportedDescription(tx.original_description ?? tx.description), + tx.cash_account_id ?? null, ) - booked.set(key, (booked.get(key) || 0) + 1) } } } catch { @@ -69,7 +102,7 @@ async function buildExistingTransactionMaps( try { const { data: unbookedBank } = await supabase .from('transactions') - .select('date, amount, original_description, description') + .select('date, amount, original_description, description, cash_account_id') .eq('company_id', companyId) .is('journal_entry_id', null) .eq('import_source', 'enable_banking') @@ -80,12 +113,13 @@ async function buildExistingTransactionMaps( for (const tx of unbookedBank) { // See booked-map note: dedup on the immutable bank original so a // user title edit cannot reopen the duplicate-import window. - const key = contentDedupKey( + addToBucket( + unbookedEnableBanking, tx.date, tx.amount, normalizeImportedDescription(tx.original_description ?? tx.description), + tx.cash_account_id ?? null, ) - unbookedEnableBanking.set(key, (unbookedEnableBanking.get(key) || 0) + 1) } } } catch { @@ -252,25 +286,57 @@ export async function ingestTransactions( continue } - // 1b. Content-based dedup: skip if an already-booked transaction - // exists with the same date, amount, and description prefix. Built from the - // normalized description so it matches the stored original_description keys. - const contentKey = contentDedupKey(raw.date, raw.amount, description) - const bookedCount = existingMaps.booked.get(contentKey) || 0 - if (bookedCount > 0) { - existingMaps.booked.set(contentKey, bookedCount - 1) - result.duplicates++ - continue + // 1b/1c. Content-dedup bridge: skip if an existing booked row (any source) + // OR an unbooked enable_banking row shares this (date, öre) bucket and a + // *bridging* description (prefix-containment, see descriptionsBridge). This + // is the net that catches re-imports the external_id check misses — chiefly + // old-format ids re-synced after the id scheme changed, and PSD2 description + // enrichment between syncs ("TIC" → "TIC BG … via internet"). Booked first, + // then unbooked, preserving the historical 1b-before-1c order. + // + // Consumed with COUNTING semantics: each match splices one stored entry out + // of its bucket, so N stored twins dedup exactly N incoming and two + // genuinely-distinct same-(date,amount) transactions are kept apart. We + // consume the LONGEST bridging stored description first so a more-specific + // twin is matched before a generic one, leaving generic entries for shorter + // incoming rows. + // + // Account guard: when BOTH the incoming batch and a stored entry have a known + // cash_account_id, they must match — so a transaction on one bank account + // never deduplicates a genuinely-different one on another account of the same + // company (the content bucket is company-wide; only external_id embeds the + // account). A null on either side falls back to bridge-allowed, leaving + // single-account and legacy (un-backfilled) rows exactly as before. + // + // Residual trade-off: within one account, a genuinely-new row whose + // description is a prefix-extension of an existing same-(date,öre) row can be + // mis-deduped; it is rare, bounded to the ~90-day PSD2 window where old-format + // ids still overlap, and the frozen external_id (Layer 1) is the exact dedup + // going forward — accepted to stop the re-import flood (the inverse, a visible + // duplicate, was the reported pain). + const bucketKey = contentBucketKey(raw.date, raw.amount) + const consumeBridgingTwin = (bucket: DescBucket): boolean => { + const entries = bucket.get(bucketKey) + if (!entries || entries.length === 0) return false + let bestIdx = -1 + let bestLen = -1 + for (let i = 0; i < entries.length; i++) { + const entry = entries[i] + const sameAccount = + cashAccountId === null || entry.cashAccountId === null || entry.cashAccountId === cashAccountId + if (sameAccount && descriptionsBridge(description, entry.desc) && entry.desc.length > bestLen) { + bestIdx = i + bestLen = entry.desc.length + } + } + if (bestIdx === -1) return false + entries.splice(bestIdx, 1) + return true } - - // 1c. Overlap dedup: skip if an unbooked enable_banking row already - // exists with the same (date, amount, description prefix). Applies to - // any incoming source — PSD2 reconnects, CSV imports over an active - // PSD2 sync, etc. Description prefix prevents unrelated transfers from - // colliding on (date, amount) alone. - const unbookedEbCount = existingMaps.unbookedEnableBanking.get(contentKey) || 0 - if (unbookedEbCount > 0) { - existingMaps.unbookedEnableBanking.set(contentKey, unbookedEbCount - 1) + if ( + consumeBridgingTwin(existingMaps.booked) || + consumeBridgingTwin(existingMaps.unbookedEnableBanking) + ) { result.duplicates++ continue }