New classification logic etc

lib/bookkeeping/__tests__/booking-templates.test.ts

@@ -249,7 +249,7 @@ describe('findMatchingTemplates', () => {
       mcc_code: 5817,
     })
     const matches = findMatchingTemplates(tx)
-    expect(matches.length).toBeLessThanOrEqual(5)
+    expect(matches.length).toBeLessThanOrEqual(20)
   })
 
   it('results are sorted by confidence descending', () => {

lib/bookkeeping/__tests__/template-embeddings.test.ts

@@ -0,0 +1,199 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest'
+import { makeTransaction, createMockSupabase } from '@/tests/helpers'
+import { BOOKING_TEMPLATES } from '../booking-templates'
+
+// Mock server-only (no-op in tests)
+vi.mock('server-only', () => ({}))
+
+// Mock OpenAI Embeddings
+vi.mock('@langchain/openai', () => {
+  class MockOpenAIEmbeddings {
+    embedQuery = vi.fn().mockResolvedValue(new Array(1536).fill(0.1))
+    embedDocuments = vi.fn().mockImplementation((texts: string[]) =>
+      Promise.resolve(texts.map(() => new Array(1536).fill(0.1)))
+    )
+  }
+  return { OpenAIEmbeddings: MockOpenAIEmbeddings }
+})
+
+// Mock Supabase
+const { supabase: mockSupabase, mockResult } = createMockSupabase()
+vi.mock('@/lib/supabase/server', () => ({
+  createServiceClient: vi.fn().mockResolvedValue(mockSupabase),
+}))
+
+describe('template-embeddings', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  describe('buildEmbeddingText', () => {
+    it('includes all relevant fields for a template', async () => {
+      const { buildEmbeddingText } = await import('../template-embeddings')
+
+      const template = BOOKING_TEMPLATES.find((t) => t.id === 'premises_rent')!
+      const text = buildEmbeddingText(template)
+
+      // Should include Swedish and English name
+      expect(text).toContain('Lokalhyra')
+      expect(text).toContain('Office rent')
+
+      // Should include description
+      expect(text).toContain(template.description_sv)
+
+      // Should include keywords
+      expect(text).toContain('hyra')
+      expect(text).toContain('lokal')
+
+      // Should include group
+      expect(text).toContain('premises')
+
+      // Should include direction
+      expect(text).toContain('utgift')
+
+      // Should include accounts
+      expect(text).toContain('5010')
+      expect(text).toContain('1930')
+    })
+
+    it('includes VAT treatment when present', async () => {
+      const { buildEmbeddingText } = await import('../template-embeddings')
+
+      const template = BOOKING_TEMPLATES.find((t) => t.id === 'premises_electricity')!
+      const text = buildEmbeddingText(template)
+
+      expect(text).toContain('standard_25')
+      expect(text).toContain('25%')
+    })
+
+    it('includes special rules when present', async () => {
+      const { buildEmbeddingText } = await import('../template-embeddings')
+
+      const template = BOOKING_TEMPLATES.find((t) => t.id === 'premises_rent')!
+      const text = buildEmbeddingText(template)
+
+      expect(text).toContain(template.special_rules_sv!)
+    })
+
+    it('includes MCC codes when present', async () => {
+      const { buildEmbeddingText } = await import('../template-embeddings')
+
+      const template = BOOKING_TEMPLATES.find((t) => t.id === 'premises_electricity')!
+      const text = buildEmbeddingText(template)
+
+      expect(text).toContain('4900')
+    })
+
+    it('includes deductibility note for non-full deductibility', async () => {
+      const { buildEmbeddingText } = await import('../template-embeddings')
+
+      const template = BOOKING_TEMPLATES.find((t) => t.deductibility === 'non_deductible')!
+      const text = buildEmbeddingText(template)
+
+      expect(text).toContain('non_deductible')
+    })
+
+    it('generates text for all 100 templates without error', async () => {
+      const { buildEmbeddingText } = await import('../template-embeddings')
+
+      for (const template of BOOKING_TEMPLATES) {
+        const text = buildEmbeddingText(template)
+        expect(text.length).toBeGreaterThan(10)
+      }
+    })
+  })
+
+  describe('buildTransactionQueryText', () => {
+    it('combines description, merchant, and direction', async () => {
+      const { buildTransactionQueryText } = await import('../template-embeddings')
+
+      const tx = makeTransaction({
+        description: 'SPOTIFY PREMIUM',
+        merchant_name: 'Spotify',
+        amount: -109,
+        mcc_code: 5815,
+      })
+
+      const text = buildTransactionQueryText(tx)
+
+      expect(text).toContain('SPOTIFY PREMIUM')
+      expect(text).toContain('Spotify')
+      expect(text).toContain('MCC 5815')
+      expect(text).toContain('utgift')
+    })
+
+    it('marks positive amounts as income', async () => {
+      const { buildTransactionQueryText } = await import('../template-embeddings')
+
+      const tx = makeTransaction({
+        description: 'Inbetalning',
+        amount: 5000,
+      })
+
+      const text = buildTransactionQueryText(tx)
+      expect(text).toContain('intäkt')
+    })
+
+    it('handles null merchant_name and mcc_code', async () => {
+      const { buildTransactionQueryText } = await import('../template-embeddings')
+
+      const tx = makeTransaction({
+        description: 'Some payment',
+        merchant_name: null,
+        mcc_code: null,
+        amount: -100,
+      })
+
+      const text = buildTransactionQueryText(tx)
+      expect(text).toContain('Some payment')
+      expect(text).toContain('utgift')
+      expect(text).not.toContain('MCC')
+    })
+  })
+
+  describe('getSchemaVersion', () => {
+    it('returns a consistent hash string', async () => {
+      const { getSchemaVersion } = await import('../template-embeddings')
+
+      const v1 = getSchemaVersion()
+      const v2 = getSchemaVersion()
+
+      expect(v1).toBe(v2)
+      expect(v1).toHaveLength(12)
+      expect(v1).toMatch(/^[a-f0-9]+$/)
+    })
+  })
+
+  describe('findSimilarTemplates', () => {
+    it('returns empty array on RPC error (graceful fallback)', async () => {
+      const { findSimilarTemplates } = await import('../template-embeddings')
+
+      // Mock staleness check
+      mockResult({ data: { schema_version: 'test' }, error: null })
+
+      const tx = makeTransaction({
+        description: 'SPOTIFY',
+        amount: -109,
+      })
+
+      // The mock will return error for the RPC call
+      mockResult({ data: null, error: { message: 'RPC failed' } })
+      const results = await findSimilarTemplates(tx)
+      expect(results).toEqual([])
+    })
+
+    it('returns empty array when no embeddings exist', async () => {
+      const { findSimilarTemplates } = await import('../template-embeddings')
+
+      mockResult({ data: [], error: null })
+
+      const tx = makeTransaction({
+        description: 'Random purchase',
+        amount: -50,
+      })
+
+      const results = await findSimilarTemplates(tx)
+      expect(results).toEqual([])
+    })
+  })
+})

lib/bookkeeping/booking-templates.ts

@@ -617,7 +617,7 @@ export const BOOKING_TEMPLATES: readonly BookingTemplate[] = [
     deductibility: 'full',
     special_rules_sv: 'Ofta utländsk leverantör (USA) med omvänd skattskyldighet',
     mcc_codes: [],
-    keywords: ['openai', 'chatgpt', 'anthropic', 'claude', 'ai', 'midjourney', 'copilot'],
+    keywords: ['openai', 'chatgpt', 'anthropic', 'claude', 'ai', 'midjourney', 'copilot', 'mistral', 'claude', 'gemini'],
     risk_level: 'NONE',
     requires_review: false,
     impact_score: 7,
@@ -663,7 +663,7 @@ export const BOOKING_TEMPLATES: readonly BookingTemplate[] = [
     vat_rate: 0.25,
     deductibility: 'full',
     mcc_codes: [5111, 5112, 5943, 5944],
-    keywords: ['kontorsmaterial', 'pennor', 'papper', 'office supplies', 'staples', 'kontorsvaror'],
+    keywords: ['kontorsmaterial', 'pennor', 'papper', 'office supplies', 'staples', 'kontorsvaror', 'kontor'],
     risk_level: 'NONE',
     requires_review: false,
     impact_score: 7,
@@ -2534,7 +2534,7 @@ export function findMatchingTemplates(
 
   return results
     .sort((a, b) => b.confidence - a.confidence)
-    .slice(0, 5)
+    .slice(0, 20)
 }
 
 /**

lib/bookkeeping/template-embeddings.ts

@@ -0,0 +1,255 @@
+/**
+ * Template Embeddings Module
+ *
+ * SERVER-ONLY: Uses OpenAI embeddings and Supabase service client.
+ *
+ * Provides semantic search over booking templates using pgvector.
+ * Templates are pre-embedded and stored in the database. Transaction
+ * text is embedded at query time and compared via cosine similarity.
+ */
+
+import 'server-only'
+import { OpenAIEmbeddings } from '@langchain/openai'
+import {
+  BOOKING_TEMPLATES,
+  getTemplateById,
+  type BookingTemplate,
+  type TemplateMatch,
+} from './booking-templates'
+import type { Transaction, EntityType } from '@/types'
+import { createHash } from 'crypto'
+
+// ============================================================
+// Constants
+// ============================================================
+
+export const EMBEDDING_MODEL = 'text-embedding-3-small'
+const EMBEDDING_LOGIC_VERSION = '1'
+const MATCH_COUNT = 20
+const MATCH_THRESHOLD = 0.5
+
+/**
+ * Schema version is a hash of the model + embedding logic version.
+ * Bump EMBEDDING_LOGIC_VERSION when buildEmbeddingText changes.
+ */
+export function getSchemaVersion(): string {
+  return createHash('sha256')
+    .update(`${EMBEDDING_MODEL}:${EMBEDDING_LOGIC_VERSION}`)
+    .digest('hex')
+    .slice(0, 12)
+}
+
+// ============================================================
+// Embedding Text Builders
+// ============================================================
+
+/**
+ * Build a rich text representation of a template for embedding.
+ * Includes all semantically relevant fields.
+ */
+export function buildEmbeddingText(template: BookingTemplate): string {
+  const parts: string[] = []
+
+  parts.push(`${template.name_sv} (${template.name_en})`)
+  parts.push(template.description_sv)
+
+  if (template.keywords.length > 0) {
+    parts.push(`Nyckelord: ${template.keywords.join(', ')}`)
+  }
+
+  parts.push(`Grupp: ${template.group}`)
+  parts.push(`Typ: ${template.direction === 'expense' ? 'utgift' : template.direction === 'income' ? 'intäkt' : 'överföring'}`)
+  parts.push(`Konton: ${template.debit_account} (debet) / ${template.credit_account} (kredit)`)
+
+  if (template.vat_treatment) {
+    parts.push(`Moms: ${template.vat_treatment} (${template.vat_rate * 100}%)`)
+  }
+
+  if (template.special_rules_sv) {
+    parts.push(`Regler: ${template.special_rules_sv}`)
+  }
+
+  if (template.mcc_codes.length > 0) {
+    parts.push(`MCC-koder: ${template.mcc_codes.join(', ')}`)
+  }
+
+  if (template.deductibility !== 'full') {
+    parts.push(`Avdragsrätt: ${template.deductibility}`)
+  }
+
+  return parts.join('. ')
+}
+
+/**
+ * Build query text from a transaction for embedding search.
+ */
+export function buildTransactionQueryText(transaction: Transaction): string {
+  const parts: string[] = []
+
+  if (transaction.description) {
+    parts.push(transaction.description)
+  }
+
+  if (transaction.merchant_name) {
+    parts.push(transaction.merchant_name)
+  }
+
+  if (transaction.mcc_code) {
+    parts.push(`MCC ${transaction.mcc_code}`)
+  }
+
+  parts.push(transaction.amount < 0 ? 'utgift' : 'intäkt')
+
+  return parts.join(' — ')
+}
+
+// ============================================================
+// Embeddings Client
+// ============================================================
+
+let embeddingsInstance: OpenAIEmbeddings | null = null
+
+function getEmbeddingsClient(): OpenAIEmbeddings {
+  if (!embeddingsInstance) {
+    embeddingsInstance = new OpenAIEmbeddings({
+      modelName: EMBEDDING_MODEL,
+      openAIApiKey: process.env.OPENAI_API_KEY,
+    })
+  }
+  return embeddingsInstance
+}
+
+// ============================================================
+// Seed All Template Embeddings
+// ============================================================
+
+export async function seedAllTemplateEmbeddings(): Promise<{
+  seeded: number
+  errors: string[]
+}> {
+  const { createServiceClient } = await import('@/lib/supabase/server')
+  const supabase = await createServiceClient()
+  const embeddings = getEmbeddingsClient()
+  const schemaVersion = getSchemaVersion()
+  const errors: string[] = []
+
+  // Build texts for all templates
+  const texts = BOOKING_TEMPLATES.map((t) => buildEmbeddingText(t))
+
+  // Batch embed all texts
+  let vectors: number[][]
+  try {
+    vectors = await embeddings.embedDocuments(texts)
+  } catch (error) {
+    return { seeded: 0, errors: [`Embedding generation failed: ${error}`] }
+  }
+
+  // Upsert each template embedding
+  let seeded = 0
+  for (let i = 0; i < BOOKING_TEMPLATES.length; i++) {
+    const template = BOOKING_TEMPLATES[i]
+    const { error } = await supabase
+      .from('booking_template_embeddings')
+      .upsert(
+        {
+          template_id: template.id,
+          embedding: JSON.stringify(vectors[i]),
+          embedding_text: texts[i],
+          model: EMBEDDING_MODEL,
+          schema_version: schemaVersion,
+        },
+        { onConflict: 'template_id' }
+      )
+
+    if (error) {
+      errors.push(`Failed to upsert ${template.id}: ${error.message}`)
+    } else {
+      seeded++
+    }
+  }
+
+  return { seeded, errors }
+}
+
+// ============================================================
+// Find Similar Templates (Semantic Search)
+// ============================================================
+
+let stalenessWarned = false
+
+export async function findSimilarTemplates(
+  transaction: Transaction,
+  entityType?: EntityType,
+  matchCount: number = MATCH_COUNT
+): Promise<TemplateMatch[]> {
+  try {
+    const { createServiceClient } = await import('@/lib/supabase/server')
+    const supabase = await createServiceClient()
+    const embeddings = getEmbeddingsClient()
+
+    // Check schema version staleness on first call
+    if (!stalenessWarned) {
+      const { data: sample } = await supabase
+        .from('booking_template_embeddings')
+        .select('schema_version')
+        .limit(1)
+        .single()
+
+      if (sample && sample.schema_version !== getSchemaVersion()) {
+        console.warn(
+          `[template-embeddings] Schema version mismatch: DB has "${sample.schema_version}", current is "${getSchemaVersion()}". Re-seed embeddings.`
+        )
+      }
+      stalenessWarned = true
+    }
+
+    // Embed the transaction query text
+    const queryText = buildTransactionQueryText(transaction)
+    const queryVector = await embeddings.embedQuery(queryText)
+
+    // Request extra results to account for post-filtering
+    const requestCount = matchCount + 10
+
+    const { data, error } = await supabase.rpc('match_booking_templates', {
+      query_embedding: JSON.stringify(queryVector),
+      match_count: requestCount,
+      match_threshold: MATCH_THRESHOLD,
+    })
+
+    if (error || !data) {
+      console.error('[template-embeddings] RPC error:', error)
+      return []
+    }
+
+    // Map RPC results to TemplateMatch[], filtering by entity type and direction
+    const isExpense = transaction.amount < 0
+    const isIncome = transaction.amount > 0
+    const results: TemplateMatch[] = []
+
+    for (const row of data as { template_id: string; similarity: number }[]) {
+      const template = getTemplateById(row.template_id)
+      if (!template) continue
+
+      // Filter by entity applicability
+      if (entityType && template.entity_applicability !== 'all' && template.entity_applicability !== entityType) {
+        continue
+      }
+
+      // Filter by direction
+      if (template.direction === 'expense' && !isExpense) continue
+      if (template.direction === 'income' && !isIncome) continue
+
+      results.push({
+        template,
+        confidence: Math.round(row.similarity * 100) / 100,
+      })
+
+      if (results.length >= matchCount) break
+    }
+
+    return results
+  } catch (error) {
+    console.error('[template-embeddings] findSimilarTemplates failed:', error)
+    return []
+  }
+}