#!/usr/bin/env npx tsx /** * Smoke test for the configured AI backend, against the real API. * * Unit tests cover which provider and model id get resolved from the * environment; they cannot tell you whether the resulting request is one the * backend accepts. This script sends real traffic over all three shapes the * app actually uses, so a credential or parameter problem surfaces here rather * than in front of a user: * * 1. A plain `messages.create` on both agent model ids. * 2a. A streamed turn with a tool, the shape the chat loop sends. * 2b. Adaptive thinking, an effort level and a cached system prompt: the * rest of that parameter set, probed separately because one turn cannot * falsify both tool use and thinking at once (see the note there). * 3. Document extraction end to end, when given a file. * * Works against whichever backend lib/ai/provider.ts resolves (AWS Bedrock or * the direct Anthropic API), so it doubles as the acceptance check when * switching between them. * * Usage: * npx tsx scripts/smoke-ai.ts * npx tsx scripts/smoke-ai.ts ./some-receipt.pdf # also runs step 3 */ import { config } from 'dotenv' config({ path: '.env.local' }) import { readFile } from 'node:fs/promises' import { basename, extname } from 'node:path' import { aiCredentialPrefix, hasAiCredentials, resolveAiProvider, } from '../lib/ai/provider' type ComposerModule = typeof import('../lib/agent/composer/client') type ExtractionModule = typeof import('../extensions/general/invoice-inbox/lib/extract-invoice-fields') let getAnthropic: ComposerModule['getAnthropic'] let EFFORT_DEEP: ComposerModule['EFFORT_DEEP'] let MAX_TOKENS_DEEP: ComposerModule['MAX_TOKENS_DEEP'] let OPUS_MODEL: ComposerModule['OPUS_MODEL'] let SONNET_MODEL: ComposerModule['SONNET_MODEL'] let extractInvoiceFields: ExtractionModule['extractInvoiceFields'] let failures = 0 function fail(step: string, err: unknown): void { const message = err instanceof Error ? err.message : String(err) console.error(` x ${step}: ${message}`) failures++ } /** Step 1: the simplest possible request, once per agent model id. */ async function ping(model: string): Promise { const start = Date.now() try { const resp = await getAnthropic().messages.create({ model, max_tokens: 10, messages: [{ role: 'user', content: 'Säg "hej" på svenska.' }], }) const text = resp.content .filter((b) => b.type === 'text') .map((b) => (b as { type: 'text'; text: string }).text) .join('') console.log(` ok ${model}: ${Date.now() - start}ms: "${text.trim()}"`) } catch (err) { fail(model, err) } } /** * Step 2: the chat loop's parameter set. * * Split in two because one turn cannot falsify both things at once. A question * that needs a tool gets answered by calling the tool, and adaptive thinking * correctly declines to reason about it, so a zero thinking-block count there * means nothing. Each probe therefore asks for one behaviour and nothing else. */ async function streamingWithTool(): Promise { const start = Date.now() try { const stream = getAnthropic().messages.stream({ model: SONNET_MODEL, max_tokens: MAX_TOKENS_DEEP, system: 'Du är en svensk redovisningsassistent. Använd verktyget när du behöver ett kontosaldo.', tools: [ { name: 'get_account_balance', description: 'Hämtar saldot för ett BAS-konto i innevarande räkenskapsår.', input_schema: { type: 'object', properties: { account: { type: 'string', description: 'BAS-kontonummer, till exempel 1930.' }, }, required: ['account'], }, }, ], messages: [{ role: 'user', content: 'Vad är saldot på konto 1930?' }], }) let deltas = 0 stream.on('text', () => { deltas++ }) const message = await stream.finalMessage() const toolUse = message.content.filter((b) => b.type === 'tool_use').length console.log( ` ok verktyg+streaming: ${Date.now() - start}ms, stop=${message.stop_reason}, ` + `textdeltan=${deltas}, tool_use=${toolUse}` ) if (toolUse === 0) { console.warn(' varning: inget tool_use-block, verktyget kan ha ignorerats') } } catch (err) { fail('verktyg+streaming', err) } } /** * Step 2b: adaptive thinking, effort and prompt caching. * * The question needs several dependent steps (reverse charge, then a partial * deduction, then which boxes move), because adaptive thinking is supposed to * skip reasoning it does not need: asking something easy cannot distinguish * "declined to think" from "parameter ignored". * * The system prompt is padded past the 1024-token minimum cacheable prefix. * Below it the API caches nothing and reports no error, so a short prompt * makes the cache counters read zero no matter whether caching works. */ async function thinkingTurn(): Promise { const start = Date.now() const filler = 'Svara alltid på svenska och hänvisa till BAS-konton med nummer. '.repeat(120) try { const stream = getAnthropic().messages.stream({ model: SONNET_MODEL, max_tokens: MAX_TOKENS_DEEP, thinking: { type: 'adaptive', display: 'summarized' }, output_config: { effort: EFFORT_DEEP }, system: [ { type: 'text', text: `Du är en svensk redovisningsassistent. ${filler}`, cache_control: { type: 'ephemeral', ttl: '1h' }, }, ], messages: [ { role: 'user', content: 'Ett svenskt momsregistrerat bolag köper en konsulttjänst från Tyskland för 10 000 kr. ' + 'Bolaget har blandad verksamhet med 60 procent avdragsrätt. Hur bokförs affären, ' + 'och vilka rutor i momsdeklarationen påverkas?', }, ], }) const message = await stream.finalMessage() const thinkingBlocks = message.content.filter((b) => b.type === 'thinking') const thinkingChars = thinkingBlocks .map((b) => (b as { type: 'thinking'; thinking: string }).thinking?.length ?? 0) .reduce((a, b) => a + b, 0) const cacheWrite = message.usage.cache_creation_input_tokens ?? 0 const cacheRead = message.usage.cache_read_input_tokens ?? 0 console.log( ` ok thinking+cache: ${Date.now() - start}ms, stop=${message.stop_reason}, ` + `thinking-block=${thinkingBlocks.length}, thinking-tecken=${thinkingChars}, ` + `cache write/read=${cacheWrite}/${cacheRead}` ) if (thinkingBlocks.length === 0) { console.warn( ' varning: inget thinking-block pa en fraga som kraver flera steg. ' + 'Chattens "Tankte…"-block skulle da aldrig fyllas.' ) } else if (thinkingChars === 0) { console.warn( ' varning: thinking-block utan text, sa display:"summarized" slog inte igenom.' ) } if (cacheWrite === 0 && cacheRead === 0) { console.warn(' varning: ingenting cachat trots prefix over minimigransen') } } catch (err) { fail('thinking+cache', err) } } /** Step 3: the real extraction path, including prompt, media block and JSON parse. */ async function extraction(path: string): Promise { const start = Date.now() const mimeByExt: Record = { '.pdf': 'application/pdf', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.png': 'image/png', '.webp': 'image/webp', '.gif': 'image/gif', } const mimeType = mimeByExt[extname(path).toLowerCase()] if (!mimeType) { fail('extraction', `okänd filändelse för ${path}, stöds: ${Object.keys(mimeByExt).join(', ')}`) return } try { const buffer = await readFile(path) const { data, rawText } = await extractInvoiceFields({ buffer, mimeType, fileName: basename(path), }) // extractInvoiceFields never throws: a null rawText means the call was // skipped or the reply did not parse, which is exactly the silent failure // this script exists to make loud. if (!rawText) { fail('extraction', 'tomt resultat (nycklar saknas, filtyp stöds inte, eller JSON-parsen föll)') return } // documentKind arrived with the receipt-aware extraction work. Read it // defensively so this script still compiles against a checkout from // before that landed: the whole point of it is to be runnable anywhere // the app runs, including an older self-hosted deployment. const kind = (data as { documentKind?: string | null }).documentKind ?? '-' console.log( ` ok extraction: ${Date.now() - start}ms, leverantör="${data.supplier.name ?? '-'}", ` + `nummer=${data.invoice.invoiceNumber ?? '-'}, datum=${data.invoice.invoiceDate ?? '-'}, ` + `typ=${kind}` ) } catch (err) { fail('extraction', err) } } async function main(): Promise { // These modules resolve their model ids at import time. Load them only after // dotenv has populated the environment so .env.local provider and model // overrides are exercised by the smoke test. const [composer, extractionModule] = await Promise.all([ import('../lib/agent/composer/client'), import('../extensions/general/invoice-inbox/lib/extract-invoice-fields'), ]) getAnthropic = composer.getAnthropic EFFORT_DEEP = composer.EFFORT_DEEP MAX_TOKENS_DEEP = composer.MAX_TOKENS_DEEP OPUS_MODEL = composer.OPUS_MODEL SONNET_MODEL = composer.SONNET_MODEL extractInvoiceFields = extractionModule.extractInvoiceFields const provider = resolveAiProvider() const explicitProvider = (process.env.AI_PROVIDER ?? '').trim().toLowerCase() console.log(`Leverantör: ${provider}`) console.log(`Nyckel: ${hasAiCredentials() ? `${aiCredentialPrefix()}…` : 'SAKNAS'}`) if (provider === 'bedrock') console.log(`Region: ${process.env.AWS_REGION || 'eu-north-1'}`) console.log(`Modeller: ${SONNET_MODEL} / ${OPUS_MODEL}`) if (!hasAiCredentials() && explicitProvider !== 'bedrock') { console.error( '\nInga synliga nycklar. Sätt ANTHROPIC_API_KEY, eller AWS_ACCESS_KEY_ID +\n' + 'AWS_SECRET_ACCESS_KEY för Bedrock. (Bedrock via instansprofil/IRSA syns\n' + 'inte härifrån: kör i så fall vidare med AI_PROVIDER=bedrock.)' ) process.exitCode = 1 return } console.log('\n1. Enkla anrop, båda modellerna') await ping(SONNET_MODEL) if (OPUS_MODEL !== SONNET_MODEL) await ping(OPUS_MODEL) console.log('\n2a. Streaming med verktyg') await streamingWithTool() console.log('\n2b. Adaptive thinking, effort och prompt-cache') await thinkingTurn() const path = process.argv[2] console.log('\n3. Dokumenttolkning') if (path) await extraction(path) else console.log(' - skipping, no file given (npx tsx scripts/smoke-ai.ts )') console.log(failures === 0 ? '\nAllt grönt.' : `\n${failures} steg föll.`) if (failures > 0) process.exitCode = 1 } main().catch((err) => { console.error(err) process.exit(1) })