diff --git a/app/api/0pt/documents/[documentId]/extraction/route.ts b/app/api/0pt/documents/[documentId]/extraction/route.ts new file mode 100644 index 0000000..746594f --- /dev/null +++ b/app/api/0pt/documents/[documentId]/extraction/route.ts @@ -0,0 +1,31 @@ +import { NextRequest, NextResponse } from "next/server"; +import { documentExtractionService } from "@/src/services/document-extraction-service"; + +export async function POST(request: NextRequest, { params }: { params: Promise<{ documentId: string }> }) { + try { + const { documentId } = await params; + const body = await request.json(); + const result = await documentExtractionService.extractDocument(documentId, { + tenant_id: body.tenant_id, + project_id: body.project_id, + actor_id: body.actor_id, + correlation_id: body.correlation_id, + model_id: body.model_id, + }); + return NextResponse.json(result, { status: 201 }); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : "Unknown error"; + return NextResponse.json({ error: message }, { status: 400 }); + } +} + +export async function GET(_request: NextRequest, { params }: { params: Promise<{ documentId: string }> }) { + try { + const { documentId } = await params; + const extractions = await documentExtractionService.listExtractionsForDocument(documentId); + return NextResponse.json(extractions); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : "Unknown error"; + return NextResponse.json({ error: message }, { status: 404 }); + } +} diff --git a/app/api/0pt/documents/capture/route.ts b/app/api/0pt/documents/capture/route.ts new file mode 100644 index 0000000..619cc15 --- /dev/null +++ b/app/api/0pt/documents/capture/route.ts @@ -0,0 +1,40 @@ +import { NextRequest, NextResponse } from "next/server"; +import { documentCaptureService } from "@/src/services/document-capture-service"; + +export async function POST(request: NextRequest) { + try { + const body = await request.json(); + const result = await documentCaptureService.captureFromMailbox( + { + mailbox: { + tenantId: body.mailbox?.tenant_id, + clientId: body.mailbox?.client_id, + clientSecret: body.mailbox?.client_secret, + mailboxAddress: body.mailbox?.mailbox_address, + folder: body.mailbox?.folder, + }, + since_iso: body.since_iso, + top: body.top, + max_pages: body.max_pages, + }, + { + tenant_id: body.tenant_id, + project_id: body.project_id, + actor_id: body.actor_id, + correlation_id: body.correlation_id, + }, + ); + return NextResponse.json(result, { status: 201 }); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : "Unknown error"; + return NextResponse.json({ error: message }, { status: 400 }); + } +} + +export async function GET(request: NextRequest) { + const { searchParams } = new URL(request.url); + const tenantId = searchParams.get("tenant_id") ?? undefined; + const projectId = searchParams.get("project_id") ?? undefined; + const documents = await documentCaptureService.listDocuments({ tenant_id: tenantId, project_id: projectId }); + return NextResponse.json(documents); +} diff --git a/app/api/0pt/route.ts b/app/api/0pt/route.ts new file mode 100644 index 0000000..b711b57 --- /dev/null +++ b/app/api/0pt/route.ts @@ -0,0 +1,19 @@ +import { NextResponse } from "next/server"; + +export async function GET() { + return NextResponse.json({ + status: "ok", + product: "0pt", + version: "1.0.0", + endpoints: [ + "/api/0pt/sources", + "/api/0pt/datasets", + "/api/0pt/data-contracts", + "/api/0pt/quality/checks", + "/api/0pt/quality/reports", + "/api/0pt/features/views", + "/api/0pt/documents/capture", + "/api/0pt/documents/{documentId}/extraction", + ], + }); +} diff --git a/app/api/0pt/sources/[sourceId]/query/route.ts b/app/api/0pt/sources/[sourceId]/query/route.ts new file mode 100644 index 0000000..f1b5d0f --- /dev/null +++ b/app/api/0pt/sources/[sourceId]/query/route.ts @@ -0,0 +1,64 @@ +/** + * app/api/0pt/sources/[sourceId]/query/route.ts + * ================================================ + * 0pt.query_source — bounded synchronous query execution for a registered + * Source. Additive: does not modify sibling ../route.ts or ../../route.ts. + * + * NOTE — in-memory scope: this module instantiates its own SourceService, + * matching the existing sources/route.ts and sources/[sourceId]/route.ts + * pattern (each API route module owns a fresh in-memory registry, so state + * is not shared across route modules yet — see the "Singleton note" in + * src/services/document-capture-service.ts for prior art on this same repo + * limitation). A source registered via POST /api/0pt/sources will not be + * resolvable here until SourceService is backed by shared persistence + * (sql/002_p0_dataset_quality.sql already defines `opt_sources` for that). + * Details: docs/P0_QUERY_SOURCE_FLOW.md. + */ +import { NextRequest, NextResponse } from "next/server"; +import { SourceService } from "@/src/services/source-service"; +import { SourceQueryService, SourceQueryTimeoutError, SourceQueryUnsupportedError } from "@/src/services/source-query-service"; + +const sourceService = new SourceService(); +const queryService = new SourceQueryService({ sourceService }); + +export async function POST(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) { + try { + const { sourceId } = await params; + const body = await request.json(); + + if (typeof body.query !== "string" || body.query.trim().length === 0) { + return NextResponse.json({ error: "query (string) is required" }, { status: 400 }); + } + if (typeof body.tenant_id !== "string" || typeof body.project_id !== "string") { + return NextResponse.json({ error: "tenant_id and project_id are required" }, { status: 400 }); + } + if (!body.connection_config || typeof body.connection_config !== "object") { + return NextResponse.json({ error: "connection_config is required" }, { status: 400 }); + } + + const result = await queryService.querySource({ + source_id: sourceId, + query: body.query, + params: body.params, + tenant_id: body.tenant_id, + project_id: body.project_id, + actor_id: body.actor_id, + correlation_id: body.correlation_id, + max_rows: body.max_rows, + timeout_ms: body.timeout_ms, + connection_config: body.connection_config, + }); + + return NextResponse.json(result, { status: 200 }); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : "Unknown error"; + if (error instanceof SourceQueryTimeoutError) { + return NextResponse.json({ error: message }, { status: 504 }); + } + if (error instanceof SourceQueryUnsupportedError) { + return NextResponse.json({ error: message }, { status: 422 }); + } + const status = /not found/i.test(message) ? 404 : 400; + return NextResponse.json({ error: message }, { status }); + } +} diff --git a/docs/P0_QUERY_SOURCE_FLOW.md b/docs/P0_QUERY_SOURCE_FLOW.md new file mode 100644 index 0000000..2c995d9 --- /dev/null +++ b/docs/P0_QUERY_SOURCE_FLOW.md @@ -0,0 +1,110 @@ +# 0pt.query_source — Bounded Source Query Flow + +Additive companion to `docs/P0_DATASET_QUALITY_FLOW.md`. Adds the capability +to actually **read data** from a registered Source, not just register its +metadata. + +## What this is + +`0pt.query_source`: given a Source already registered via `0pt.create_source` +(`source_type = "database"`, `adapter_id = "fabric_onelake_sql"`), run a +caller-supplied SQL query against the customer's own Azure Fabric / OneLake +SQL analytics endpoint and return rows — synchronously, bounded by a max-row +and max-time ceiling. + +**Tenant-agnostic by construction.** Any SIAX customer registers their own +Fabric/OneLake warehouse (their own Entra tenant, their own service +principal, their own SQL endpoint). Nothing in the adapter or service layer +is hardcoded to one customer — every credential and endpoint is a parameter +supplied per call, never an env var fallback. + +## Why bounded + synchronous, not just "run any query" + +This path is for small, interactive/preview-style reads (validate a Source +is reachable, sample a table, preview a query before scheduling it). It is +*not* a general query engine: + +- `max_rows`: default 1,000, hard ceiling 10,000 (`SOURCE_QUERY_DEFAULT_MAX_ROWS` / + `SOURCE_QUERY_HARD_MAX_ROWS` in `source-query-service.ts`). Excess rows are + truncated and the response is flagged `truncated: true`. +- `timeout_ms`: default 10s, hard ceiling 30s (`SOURCE_QUERY_DEFAULT_TIMEOUT_MS` / + `SOURCE_QUERY_HARD_MAX_TIMEOUT_MS`). A caller-requested value above the + ceiling is clamped down, not rejected. +- Anything heavier — large exports, long scans, scheduled/recurring pulls — + must go through **N0D as a job** instead, the same split + `quality-check-service.ts` already uses (`runQualityChecks` → CL0UD policy + → N0D job, not an inline synchronous call). + +## Architecture + +``` +Adapter: src/adapters/fabric-onelake-adapter.ts + - acquireFabricAccessToken(): Entra service-principal OAuth2 + client-credentials flow (POST login.microsoftonline.com/{tenantId} + /oauth2/v2.0/token) — generalized from v0-datahub's + scripts/ops/cloud-bootstrap.mjs `aadToken()`. + - FabricOneLakeAdapter.query(): opens a TDS pool to the Fabric SQL + analytics endpoint using that token (azure-active-directory-access-token + auth), same shape as v0-datahub's lib/warehouse/fabric-sql.ts + `buildConfig()`/`runFabricQuery()`, generalized to caller-supplied + tenant/client/secret/endpoint/database instead of one customer's env vars. + +Service: src/services/source-query-service.ts + - SourceQueryService.querySource(): looks up the Source via the existing + SourceService.getSource() (read-only — does not modify source-service.ts), + validates source_type/adapter_id, clamps max_rows/timeout_ms, races the + adapter query against a timeout, truncates rows, emits an AUD0 event, + always releases the adapter's connection. + +Route: app/api/0pt/sources/[sourceId]/query/route.ts + - POST { query, params?, tenant_id, project_id, actor_id?, correlation_id?, + max_rows?, timeout_ms?, connection_config } -> QuerySourceResult + - 400 on missing query/tenant_id/project_id/connection_config + - 404 when the source does not exist + - 422 (SourceQueryUnsupportedError) when source_type != "database" or + adapter_id != "fabric_onelake_sql" + - 504 (SourceQueryTimeoutError) on timeout + +AUD0: event_type "0pt.source_queried", capability_id "0pt.query_source", + resource_type "source", risk_level "L4" (live external data access with a + tenant's own credentials — one level above the "L3" used for Source/ + Dataset/QualityCheck metadata writes, matching the "L4 = restricted + sensitivity" precedent in src/data-platform/services/catalog-service.ts). + Emitted on both success (ok:true, row_count, truncated, duration_ms) and + failure (ok:false, error) via the same `Aud0Client` pattern as + source-service.ts / quality-check-service.ts. +``` + +## Known gaps / deliberately left out + +- **`mssql` is not a declared dependency yet.** package.json and + pnpm-lock.yaml already have unrelated uncommitted changes from a separate + in-flight session at the time this capability was built, so adding a new + dependency there was avoided to prevent a collision. The adapter loads + `mssql` lazily via a non-literal dynamic import (works with any bundler/ + typecheck without the package installed) and throws a clear, actionable + error if it's missing. Once package.json is settled, `pnpm add mssql` is + the only change needed to make `query()` hit a real Fabric endpoint — no + code changes required. +- **In-memory SourceService is not shared across API route modules.** + Same pre-existing limitation as `sources/route.ts` / + `sources/[sourceId]/route.ts` (each Next.js route module instantiates its + own `SourceService`, so a Source created via `POST /api/0pt/sources` is not + yet visible to `POST /api/0pt/sources/[sourceId]/query` in the same running + process). `document-capture-service.ts` hit the same issue and added a + shared singleton for its own two routes; doing the same for Source would + mean editing `source-service.ts` and/or the two existing `sources/route.ts` + files, which was out of scope here to avoid touching that flagged area. + The real fix is the same one already pointed at by `sql/002_p0_dataset_quality.sql` + (`opt_sources` table) — back `SourceService` with real persistence. +- **No CL0UD policy check before executing a query.** `quality-check-service.ts` + calls `Cl0udClient.evaluatePolicy()` before creating a N0D job; this + synchronous path does not, since it is scoped as a bounded preview read, not + a governed job. Worth revisiting once policy-gating on `0pt.query_source` + is defined. +- **No SQL rewriting/injection of `TOP`/`LIMIT`.** Row bounding is applied to + whatever the adapter returns, not pushed into the query itself (arbitrary + caller SQL can't be safely rewritten). A query that itself returns far more + rows than `max_rows` still executes in full on the warehouse before being + truncated client-side — only the *response* is bounded, not warehouse-side + compute. Documented, not silently hidden. diff --git a/src/adapters/document-intelligence-adapter.ts b/src/adapters/document-intelligence-adapter.ts new file mode 100644 index 0000000..e841b99 --- /dev/null +++ b/src/adapters/document-intelligence-adapter.ts @@ -0,0 +1,208 @@ +/** + * src/adapters/document-intelligence-adapter.ts + * ================================================ + * Tenant-agnostic Azure AI Document Intelligence client. + * + * Generalized from v0-datahub/lib/invoice-inbox/doc-intelligence.ts, which + * hardcoded the `prebuilt-invoice` model and a fixed invoice-shaped return + * type (amount_gross, vendor, vat, ...). Here `modelId` is a CALLER-SUPPLIED + * parameter — "prebuilt-invoice", "prebuilt-receipt", "prebuilt-layout", + * "prebuilt-read", or any custom-trained model id — and the adapter returns + * a generic field map instead of assuming any particular document shape. + * + * Evidence discipline (correctness-critical, not a style choice): a field is + * only ever populated with the value + confidence Document Intelligence + * actually reported. If DI did not return a value, the field comes back as + * { value: null, confidence: , found: false }. + * Nothing here ever defaults, guesses, or backfills a value. + */ + +export interface DocumentIntelligenceConfig { + endpoint: string; + apiKey: string; + /** Default: "2024-11-30" — the API version the production invoice pipeline used. */ + apiVersion?: string; + /** Delay between result-polling attempts, ms. Default 2000. (Tests override to run fast.) */ + pollIntervalMs?: number; + /** Delay before retrying after a 429, ms. Default 6000. (Tests override to run fast.) */ + rateLimitRetryMs?: number; + /** Max result-polling attempts before timing out. Default 40. */ + maxPollAttempts?: number; + /** Max analyze-request attempts (each 429 counts as one). Default 4. */ + maxAnalyzeAttempts?: number; +} + +export interface ExtractedField { + value: string | number | boolean | null; + confidence: number | null; + found: boolean; +} + +export interface DocumentIntelligenceResult { + modelId: string; + apiVersion: string; + /** Full extracted text, when the API returns one (present across prebuilt models). */ + content: string | null; + /** Top-level structured fields (documents[0].fields), normalized. Empty for pure layout/read. */ + fields: Record; + /** Array-of-object fields (e.g. invoice/receipt line items), each item pre-normalized. */ + arrayFields: Record>>; + /** documents[0].confidence, when the model reports a whole-document confidence. */ + documentConfidence: number | null; + pageCount: number | null; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** Normalize one Document Intelligence field object into an evidence-based ExtractedField. */ +function normalizeField(raw: unknown): ExtractedField { + const f = raw as Record | null | undefined; + if (!f) return { value: null, confidence: null, found: false }; + let value: string | number | boolean | null = null; + switch (f.type) { + case "string": + value = (f.valueString as string | undefined) ?? (f.content as string | undefined) ?? null; + break; + case "number": + value = (f.valueNumber as number | undefined) ?? null; + break; + case "integer": + value = (f.valueInteger as number | undefined) ?? null; + break; + case "date": + value = (f.valueDate as string | undefined) ?? null; + break; + case "time": + value = (f.valueTime as string | undefined) ?? null; + break; + case "phoneNumber": + value = (f.valuePhoneNumber as string | undefined) ?? null; + break; + case "countryRegion": + value = (f.valueCountryRegion as string | undefined) ?? null; + break; + case "selectionMark": + value = (f.valueSelectionMark as string | undefined) ?? null; + break; + case "currency": { + const cur = f.valueCurrency as Record | undefined; + value = typeof cur?.amount === "number" ? (cur.amount as number) : null; + break; + } + case "boolean": + value = typeof f.valueBoolean === "boolean" ? (f.valueBoolean as boolean) : null; + break; + default: + value = (f.valueString as string | undefined) ?? (f.content as string | undefined) ?? null; + } + const found = value !== null && value !== undefined && value !== ""; + const confidence = typeof f.confidence === "number" ? (f.confidence as number) : null; + return { value: found ? value : null, confidence, found }; +} + +function normalizeObjectFields(obj: Record | null | undefined): Record { + const out: Record = {}; + for (const [key, raw] of Object.entries(obj ?? {})) { + out[key] = normalizeField(raw); + } + return out; +} + +export class DocumentIntelligenceAdapter { + constructor(private config: DocumentIntelligenceConfig) {} + + static isConfigured(config: Partial): boolean { + return !!(config.endpoint && config.apiKey); + } + + /** + * Analyze a document buffer with the given model. Throws on any failure — + * callers must never interpret a caught error as "no fields found". A + * genuinely empty field is represented by { found: false } *inside a + * successful result*; a failed call must propagate as an error, not a + * fabricated empty success. + */ + async analyzeDocument(buf: Buffer, modelId: string): Promise { + const apiVersion = this.config.apiVersion || "2024-11-30"; + const base = this.config.endpoint.replace(/\/$/, ""); + const analyzeUrl = `${base}/documentintelligence/documentModels/${encodeURIComponent(modelId)}:analyze?api-version=${apiVersion}`; + + const maxAnalyzeAttempts = this.config.maxAnalyzeAttempts ?? 4; + const rateLimitRetryMs = this.config.rateLimitRetryMs ?? 6000; + const pollIntervalMs = this.config.pollIntervalMs ?? 2000; + const maxPollAttempts = this.config.maxPollAttempts ?? 40; + + let operationLocation = ""; + for (let attempt = 0; attempt < maxAnalyzeAttempts; attempt++) { + const res = await fetch(analyzeUrl, { + method: "POST", + headers: { "Ocp-Apim-Subscription-Key": this.config.apiKey, "Content-Type": "application/json" }, + body: JSON.stringify({ base64Source: buf.toString("base64") }), + }); + if (res.status === 429) { + await sleep(rateLimitRetryMs); + continue; + } + if (res.status !== 202) { + const text = await res.text().catch(() => ""); + throw new Error(`Document Intelligence analyze request failed: ${res.status} ${text.slice(0, 300)}`); + } + operationLocation = res.headers.get("operation-location") || ""; + break; + } + if (!operationLocation) { + throw new Error( + "Document Intelligence analyze did not return an operation-location (rate-limited after retries, or misconfigured)", + ); + } + + let payload: Record | null = null; + for (let i = 0; i < maxPollAttempts; i++) { + await sleep(pollIntervalMs); + const pollRes = await fetch(operationLocation, { headers: { "Ocp-Apim-Subscription-Key": this.config.apiKey } }); + const json = (await pollRes.json()) as Record; + if (json.status === "succeeded") { + payload = json; + break; + } + if (json.status === "failed") { + throw new Error(`Document Intelligence analyze failed: ${JSON.stringify(json.error ?? json).slice(0, 500)}`); + } + } + if (!payload) { + throw new Error("Document Intelligence analyze timed out waiting for a result"); + } + + const analyzeResult = (payload.analyzeResult as Record) ?? {}; + const documents = analyzeResult.documents as Array> | undefined; + const doc = documents?.[0]; + const rawFields = (doc?.fields as Record) ?? {}; + + const fields: Record = {}; + const arrayFields: Record>> = {}; + for (const [key, raw] of Object.entries(rawFields)) { + const f = raw as Record; + if (f?.type === "array" && Array.isArray(f.valueArray)) { + arrayFields[key] = (f.valueArray as Array>).map((item) => + normalizeObjectFields(item?.valueObject as Record | undefined), + ); + } else { + fields[key] = normalizeField(f); + } + } + + const pages = analyzeResult.pages as unknown[] | undefined; + + return { + modelId, + apiVersion, + content: typeof analyzeResult.content === "string" ? (analyzeResult.content as string) : null, + fields, + arrayFields, + documentConfidence: typeof doc?.confidence === "number" ? (doc.confidence as number) : null, + pageCount: Array.isArray(pages) ? pages.length : null, + }; + } +} diff --git a/src/adapters/fabric-onelake-adapter.ts b/src/adapters/fabric-onelake-adapter.ts new file mode 100644 index 0000000..af7c13f --- /dev/null +++ b/src/adapters/fabric-onelake-adapter.ts @@ -0,0 +1,223 @@ +/** + * src/adapters/fabric-onelake-adapter.ts + * ======================================== + * Tenant-agnostic Azure Fabric / OneLake SQL analytics endpoint adapter + * (adapter_id = "fabric_onelake_sql" on a 0pt Source). + * + * Generalized from two proven patterns in v0-datahub: + * - scripts/ops/cloud-bootstrap.mjs `aadToken()` — Entra service-principal + * OAuth2 client-credentials flow (POST login.microsoftonline.com/{tenant} + * /oauth2/v2.0/token, grant_type=client_credentials). + * - lib/warehouse/fabric-sql.ts `buildConfig()` / `runFabricQuery()` — using + * that token as an `azure-active-directory-access-token` credential on a + * `mssql` TDS connection to the Fabric SQL analytics endpoint, and + * returning `result.recordset` as plain rows. + * + * Both of those originals hardcode ONE customer's tenant/client/secret/ + * endpoint as module-level constants or env vars (AZ_SP_*, FABRIC_SQL_ENDPOINT, + * kv-pts-datahub, ...). Nothing here is hardcoded: tenantId, clientId, + * clientSecret, sqlEndpoint and database are always caller-supplied, because + * this is a SIAX platform capability — any SIAX customer registers their own + * Fabric/OneLake warehouse as a Source and queries it through this adapter. + * + * Dependency note: `mssql` (the TDS driver used by fabric-sql.ts) is NOT a + * declared dependency of this repo yet, and package.json/pnpm-lock.yaml + * already have unrelated uncommitted changes from a separate in-flight + * session — so it is deliberately not added here to avoid colliding with + * that work. The driver is loaded lazily via a non-literal dynamic import + * (see `loadDriver`) so: + * - this file has zero hard dependency at import/bundle time (Next.js/ + * webpack cannot statically resolve a non-literal specifier, so it does + * not fail the build even though "mssql" isn't installed), and + * - `tsc --noEmit` does not attempt to resolve module types for it either + * (a widened `string`-typed specifier makes `import(...)` type as + * `Promise` instead of triggering module resolution). + * Once `mssql` is added to package.json (`pnpm add mssql`), query() works + * against a real endpoint with no code change here. Until then, `query()` + * throws a clear, actionable error — see docs/P0_QUERY_SOURCE_FLOW.md. + */ + +const DEFAULT_SQL_SCOPE = "https://database.windows.net/.default"; +const TOKEN_REFRESH_SKEW_MS = 60_000; +const DEFAULT_REQUEST_TIMEOUT_MS = 30_000; + +export interface FabricOneLakeCredentials { + /** Entra (Azure AD) tenant id the service-principal app registration lives in. */ + tenantId: string; + /** Entra app (client) id — app-only, client-credentials grant. */ + clientId: string; + /** Entra app client secret. Never logged, never persisted by this adapter. */ + clientSecret: string; +} + +export interface FabricOneLakeConnectionConfig { + credentials: FabricOneLakeCredentials; + /** Fabric SQL analytics endpoint host, e.g. "xxxx.datawarehouse.fabric.microsoft.com" (scheme optional). */ + sqlEndpoint: string; + /** Target database / lakehouse SQL endpoint name, e.g. "lh_customer_datahub". */ + database: string; + /** AAD scope used for the SQL token. Defaults to the Azure SQL / Fabric warehouse resource scope. */ + scope?: string; + /** Per-request timeout passed to the mssql pool config. Default 30s. */ + requestTimeoutMs?: number; +} + +export interface FabricAccessToken { + accessToken: string; + expiresAtEpochMs: number; +} + +export interface AdapterQueryResult> { + rows: T[]; + rowCount: number; + durationMs: number; +} + +/** + * OAuth2 client-credentials token acquisition for an Azure AD service + * principal — generalized 1:1 from v0-datahub's `aadToken()`. Exported + * standalone (not just as a private method) so it can be unit-tested and + * reused by callers that only need a token (e.g. to probe connectivity) + * without opening a SQL connection. + */ +export async function acquireFabricAccessToken( + credentials: FabricOneLakeCredentials, + scope: string = DEFAULT_SQL_SCOPE, + fetchImpl: typeof fetch = fetch, +): Promise { + const { tenantId, clientId, clientSecret } = credentials; + if (!tenantId || !clientId || !clientSecret) { + throw new Error("fabric_onelake_adapter: tenantId, clientId and clientSecret are all required"); + } + const body = new URLSearchParams({ + client_id: clientId, + client_secret: clientSecret, + grant_type: "client_credentials", + scope, + }); + const res = await fetchImpl(`https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body, + }); + const json: { access_token?: string; expires_in?: number; error?: string; error_description?: string } = await res.json(); + if (!res.ok || !json.access_token) { + throw new Error( + `fabric_onelake_adapter: AAD token request failed (scope=${scope}): ${json.error ?? res.status} ${json.error_description ?? ""}`.trim(), + ); + } + const expiresInMs = (typeof json.expires_in === "number" ? json.expires_in : 3600) * 1000; + return { accessToken: json.access_token, expiresAtEpochMs: Date.now() + expiresInMs }; +} + +function stripScheme(endpoint: string): string { + return endpoint.replace(/^https?:\/\//, "").replace(/\/+$/, ""); +} + +/** Minimal shape of the pieces of the `mssql` module this adapter actually uses. Kept as a structural type, never imported from the real package (see file header). */ +interface MssqlLike { + ConnectionPool: new (config: Record) => { + connect(): Promise; + request(): { + input(name: string, value: unknown): unknown; + query(text: string): Promise<{ recordset?: unknown[] }>; + }; + close(): Promise; + }; +} + +/** + * Adapter for Source.adapter_id === "fabric_onelake_sql". One instance holds + * one registered Source's connection config; nothing is shared across + * tenants/customers, and no config here ever falls back to an env var. + */ +export class FabricOneLakeAdapter { + private token: FabricAccessToken | null = null; + private pool: InstanceType | null = null; + + constructor( + private readonly config: FabricOneLakeConnectionConfig, + private readonly fetchImpl: typeof fetch = fetch, + ) {} + + private async getToken(): Promise { + if (this.token && this.token.expiresAtEpochMs - TOKEN_REFRESH_SKEW_MS > Date.now()) { + return this.token.accessToken; + } + this.token = await acquireFabricAccessToken(this.config.credentials, this.config.scope, this.fetchImpl); + return this.token.accessToken; + } + + /** + * Loads the `mssql` TDS driver at call time via a *non-literal* specifier. + * `moduleName` is deliberately widened to `string` (not the literal + * `"mssql"`) so TypeScript types the result as `Promise` instead of + * resolving module types, and so bundlers cannot statically inline the + * specifier into a build-time-resolved chunk. Overridable for tests. + */ + private async loadDriver(): Promise { + const moduleName: string = "mssql"; + try { + const mod = await import(moduleName); + return (mod.default ?? mod) as MssqlLike; + } catch (err) { + throw new Error( + `fabric_onelake_adapter: "mssql" package is not installed/resolvable. Add it to package.json ` + + `(dependencies.mssql) to enable live Fabric SQL query execution. Original error: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + } + + private async getPool(): Promise> { + if (this.pool) return this.pool; + const sql = await this.loadDriver(); + const token = await this.getToken(); + const poolConfig = { + server: stripScheme(this.config.sqlEndpoint), + database: this.config.database, + options: { encrypt: true, trustServerCertificate: false }, + authentication: { + type: "azure-active-directory-access-token", + options: { token }, + }, + pool: { max: 1, min: 0, idleTimeoutMillis: 30_000 }, + requestTimeout: this.config.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS, + }; + const pool = new sql.ConnectionPool(poolConfig); + await pool.connect(); + this.pool = pool; + return pool; + } + + /** + * Execute a query and return rows + timing. This adapter does NOT enforce + * row/time bounds itself — that is the job of the caller (see + * src/services/source-query-service.ts), which wraps this in a timeout and + * truncates the result for the synchronous path. + */ + async query>(queryText: string, params?: Record): Promise> { + const start = Date.now(); + const pool = await this.getPool(); + const request = pool.request(); + if (params) { + for (const [key, value] of Object.entries(params)) { + request.input(key, value); + } + } + const result = await request.query(queryText); + const rows = (result.recordset ?? []) as T[]; + return { rows, rowCount: rows.length, durationMs: Date.now() - start }; + } + + async close(): Promise { + if (this.pool) { + await this.pool.close(); + this.pool = null; + } + } +} + +export function createFabricOneLakeAdapter(config: FabricOneLakeConnectionConfig, fetchImpl?: typeof fetch): FabricOneLakeAdapter { + return new FabricOneLakeAdapter(config, fetchImpl); +} diff --git a/src/adapters/graph-mailbox-adapter.ts b/src/adapters/graph-mailbox-adapter.ts new file mode 100644 index 0000000..2e23e5f --- /dev/null +++ b/src/adapters/graph-mailbox-adapter.ts @@ -0,0 +1,188 @@ +/** + * src/adapters/graph-mailbox-adapter.ts + * ====================================== + * Tenant-agnostic Microsoft Graph mailbox poller (app-only client-credentials). + * + * Generalized from v0-datahub/lib/invoice-inbox/poll.ts: there the mailbox + * (invoice@protosell.se), the Entra app, and the folder were all hardcoded + * module constants. Here every one of those is a CALLER-SUPPLIED parameter — + * any SIAX customer can register any mailbox (invoices, HR forms, contracts, + * receipts, ...) and poll it through this adapter. Nothing about invoices, + * vendors, or Protosell is baked in here — that logic (vendor-kb matching, + * dup-hash, DQ scoring, invoice classification) was intentionally left + * behind; it does not belong in a generic capture adapter. + */ + +const GRAPH_BASE = "https://graph.microsoft.com/v1.0"; + +export interface GraphMailboxConfig { + /** Entra (Azure AD) tenant id the app registration lives in. */ + tenantId: string; + /** Entra app (client) id — app-only, client-credentials grant. */ + clientId: string; + /** Entra app client secret. Never logged, never persisted by this adapter. */ + clientSecret: string; + /** Mailbox address to poll, e.g. "invoices@customer.com". */ + mailboxAddress: string; + /** Mail folder to poll. Default: "inbox". */ + folder?: string; +} + +export interface GraphMessage { + id: string; + internetMessageId: string | null; + conversationId: string | null; + receivedAt: string | null; + senderName: string | null; + senderEmail: string | null; + toRecipients: string[]; + subject: string | null; + bodyPreview: string | null; + /** HTML body stripped to plain text. Falls back to bodyPreview when body is absent. */ + bodyText: string | null; + hasAttachments: boolean; + webLink: string | null; +} + +export interface GraphAttachment { + id: string; + name: string | null; + contentType: string | null; + sizeBytes: number | null; + isPdf: boolean; + contentBytes: Buffer; +} + +export interface PollMessagesOptions { + /** Only return messages received at/after this ISO timestamp (incremental polling). */ + sinceISO?: string; + /** Page size per Graph request. Default 50. */ + top?: number; + /** Max number of pages to walk. Default 2 (raise for backfills). */ + maxPages?: number; +} + +/** Strip HTML to plain text. Generic — no document-type-specific parsing. */ +function stripHtml(html: string): string { + return (html || "") + .replace(//gi, " ") + .replace(//gi, " ") + .replace(/<[^>]+>/g, " ") + .replace(/ /gi, " ") + .replace(/&/gi, "&") + .replace(/</gi, "<") + .replace(/>/gi, ">") + .replace(/\s+/g, " ") + .trim(); +} + +export class GraphMailboxAdapter { + private cachedToken: { token: string; expiresAt: number } | null = null; + + constructor(private config: GraphMailboxConfig) {} + + static isConfigured(config: Partial): boolean { + return !!(config.tenantId && config.clientId && config.clientSecret && config.mailboxAddress); + } + + private async getToken(): Promise { + if (this.cachedToken && this.cachedToken.expiresAt > Date.now() + 30_000) { + return this.cachedToken.token; + } + const { tenantId, clientId, clientSecret } = this.config; + if (!tenantId || !clientId || !clientSecret) { + throw new Error("GraphMailboxAdapter: missing tenantId/clientId/clientSecret"); + } + const body = new URLSearchParams({ + client_id: clientId, + client_secret: clientSecret, + grant_type: "client_credentials", + scope: "https://graph.microsoft.com/.default", + }); + const res = await fetch(`https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body, + }); + const json: Record = await res.json(); + if (!res.ok) { + throw new Error(`Graph token request failed: ${json.error || res.status} ${json.error_description || ""}`); + } + const expiresInMs = (typeof json.expires_in === "number" ? json.expires_in : 3600) * 1000; + this.cachedToken = { token: json.access_token as string, expiresAt: Date.now() + expiresInMs }; + return this.cachedToken.token; + } + + private async get(path: string): Promise { + const token = await this.getToken(); + const url = path.startsWith("http") ? path : `${GRAPH_BASE}${path}`; + const res = await fetch(url, { headers: { authorization: `Bearer ${token}` } }); + const json: any = await res.json(); + if (!res.ok) { + throw new Error(`Graph GET ${path} -> ${json.error?.code || res.status}: ${json.error?.message || ""}`); + } + return json; + } + + /** List messages in the configured mailbox/folder, newest first, optionally incremental. */ + async pollMessages(opts: PollMessagesOptions = {}): Promise { + const folder = this.config.folder || "inbox"; + const top = opts.top ?? 50; + const maxPages = opts.maxPages ?? 2; + const select = + "$select=id,internetMessageId,conversationId,receivedDateTime,from,toRecipients,subject,bodyPreview,hasAttachments,webLink,body"; + let sinceFilter = ""; + if (opts.sinceISO) { + sinceFilter = `&$filter=receivedDateTime ge ${new Date(opts.sinceISO).toISOString()}`; + } + let url = + `${GRAPH_BASE}/users/${encodeURIComponent(this.config.mailboxAddress)}/mailFolders/${encodeURIComponent(folder)}` + + `/messages?$top=${top}&$orderby=receivedDateTime desc&${select}${sinceFilter}`; + + const out: GraphMessage[] = []; + for (let page = 0; page < maxPages && url; page++) { + const data = await this.get(url); + for (const m of data.value || []) { + out.push({ + id: m.id, + internetMessageId: m.internetMessageId || null, + conversationId: m.conversationId || null, + receivedAt: m.receivedDateTime || null, + senderName: m.from?.emailAddress?.name || null, + senderEmail: m.from?.emailAddress?.address || null, + toRecipients: (m.toRecipients || []).map((r: any) => r.emailAddress?.address).filter(Boolean), + subject: m.subject || null, + bodyPreview: m.bodyPreview || null, + bodyText: m.body?.content ? stripHtml(m.body.content) : m.bodyPreview || null, + hasAttachments: !!m.hasAttachments, + webLink: m.webLink || null, + }); + } + url = data["@odata.nextLink"] || ""; + } + return out; + } + + /** List and download every file attachment on a message. */ + async listAttachments(messageId: string): Promise { + const data = await this.get( + `/users/${encodeURIComponent(this.config.mailboxAddress)}/messages/${messageId}/attachments`, + ); + const out: GraphAttachment[] = []; + for (const a of data.value || []) { + if (a["@odata.type"] !== "#microsoft.graph.fileAttachment" || !a.contentBytes) continue; + const contentType: string | null = a.contentType || null; + const name: string | null = a.name || null; + const isPdf = (contentType || "").includes("pdf") || /\.pdf$/i.test(name || ""); + out.push({ + id: a.id, + name, + contentType, + sizeBytes: typeof a.size === "number" ? a.size : null, + isPdf, + contentBytes: Buffer.from(a.contentBytes, "base64"), + }); + } + return out; + } +} diff --git a/src/adapters/local-blob-store.ts b/src/adapters/local-blob-store.ts new file mode 100644 index 0000000..a2f4582 --- /dev/null +++ b/src/adapters/local-blob-store.ts @@ -0,0 +1,47 @@ +/** + * src/adapters/local-blob-store.ts + * ================================== + * INTERIM local-disk blob store — NOT the real ST0RE bytes backend. + * + * KNOWN GAP: src/clients/st0re-client.ts only registers artifact METADATA + * (a pointer: provider/bucket/key/region). It has no endpoint that accepts + * and persists raw bytes — ST0RE does not yet expose a real object-storage + * write path for 0pt to call. Until it does, document-capture-service.ts + * and document-extraction-service.ts write attachment/extraction bytes here + * so the capability is functionally real end-to-end (download → persist → + * read back for extraction) instead of silently discarding the PDF bytes + * and only faking a metadata pointer. + * + * This is single-node, unencrypted-at-rest beyond the host filesystem, and + * has no retention/lifecycle management — do not treat it as production + * object storage. Swap it for a real ST0RE (or direct MinIO/Azure Blob) + * client the day one exists; every call site that constructs this class is + * marked "INTERIM" so the seam is easy to find. + */ +import { promises as fs } from "fs"; +import path from "path"; + +export interface LocalBlobWriteResult { + provider: "local_interim"; + bucket: string; + key: string; + absolutePath: string; + sizeBytes: number; +} + +export class LocalBlobStore { + constructor(private rootDir: string = process.env.OPT_LOCAL_BLOB_DIR || path.join(process.cwd(), ".data", "artifacts")) {} + + async write(bucket: string, key: string, bytes: Buffer): Promise { + const safeKey = key.replace(/\.\./g, "_"); + const absolutePath = path.join(this.rootDir, bucket, safeKey); + await fs.mkdir(path.dirname(absolutePath), { recursive: true }); + await fs.writeFile(absolutePath, bytes); + return { provider: "local_interim", bucket, key: safeKey, absolutePath, sizeBytes: bytes.length }; + } + + async read(bucket: string, key: string): Promise { + const absolutePath = path.join(this.rootDir, bucket, key); + return fs.readFile(absolutePath); + } +} diff --git a/src/adapters/odoo-bills-adapter.ts b/src/adapters/odoo-bills-adapter.ts new file mode 100644 index 0000000..d6995d6 --- /dev/null +++ b/src/adapters/odoo-bills-adapter.ts @@ -0,0 +1,279 @@ +/** + * src/adapters/odoo-bills-adapter.ts + * ===================================== + * `WritebackAdapter` (../adapters/writeback-adapter.ts) implementation for + * Odoo Vendor Bills (`account.move`, `move_type: 'in_invoice'`) -- + * registered under `adapterId: "odoo_bills"` onto an + * `ExternalWritebackService` (../services/external-writeback-service.ts). + * + * PORTED FROM v0-datahub's `lib/odoo/invoice-bridge.ts` (read in full before + * writing this file), generalized as one WritebackAdapter implementation + * rather than a standalone module with its own Supabase-coupled pipeline + * function (`processInboxToOdoo`). What is a DIRECT PORT of proven logic, + * vs. what is NEW/GENERALIZED, stated precisely: + * + * DIRECT PORT (same behavior, same guard order, same reasoning): + * - `isReminder`/`duplicateFlag` guard -- invoice-bridge.ts's + * `processInboxToOdoo`: "if (invoice.is_reminder || invoice.duplicate_flag) + * return { ok:false, error: ... }" (a reminder or a detected duplicate is + * NEVER posted to Odoo). Ported verbatim as this adapter's first guard in + * `write()`. + * - "already posted, skip" guard -- invoice-bridge.ts: "if + * (invoice.odoo_move_id) { ...hoppar över...; return { ok:true, odoo_id: + * invoice.odoo_move_id } }". Ported as the `existingExternalId` check -- + * this adapter's own idempotency defense-in-depth layer, complementing + * (not replacing) `ExternalWritebackService`'s own in-memory claim -- see + * that service's own module doc comment and writeback-adapter.ts's + * `WritebackAdapter.write` doc comment for why BOTH layers exist. + * - Partner matching by org-nr -- invoice-bridge.ts's + * `matchInvoiceToOdooPartner`: digits-only comparison against + * `odoo_export__res_partner.vat` so "556xxx-xxxx", "SE556xxxxxxxx01", and + * "5561234567" all match the same partner. Ported as + * `matchPartnerViaFabric` below, with ONE necessary substitution: the + * original queried via v0-datahub's own `@/lib/warehouse/client` + * (Next.js-app-local, not importable from this repo) -- this port uses + * 0pt's own already-built, equally generic + * `../adapters/fabric-onelake-adapter.ts` (built this same pass, for + * 0pt.document_ocr's own needs) against the SAME table name + * (`odoo_export__res_partner`) and the SAME digits-only `LIKE` query + * shape, parameterized instead of string-interpolated. + * - Vendor Bill request shape -- invoice-bridge.ts's `pushInvoiceToOdoo`: + * POSTs `{ model: "account.move", invoice: { partner_id, ref, + * invoice_date, invoice_date_due, invoice_line_ids: [...] } }` to + * `/api/odoo/invoices/write`. Ported verbatim as this adapter's own POST + * body -- same endpoint path convention, same field names, so an existing + * `/api/odoo/invoices/write` deployment needs no change to accept calls + * from this adapter. + * - Line synthesis fallback -- invoice-bridge.ts's `processInboxToOdoo` + * builds a single summary `invoice_line_ids` entry + * (`description: subject||vendor||"Faktura {number}"`, `quantity: 1, + * unit_price: amount_excl_vat`) when no itemized lines exist. Ported as + * `synthesizeSummaryLine` below, used only when `payload.lines` is absent. + * + * GENERALIZED / NEW (did not exist in the original, or could not be ported + * 1:1): + * - The Supabase read/claim/update-status statements in + * `processInboxToOdoo` (the `invoice_inbox` table CRUD, and its own + * atomic `UPDATE ... WHERE odoo_move_id IS NULL` claim) are NOT ported + * here -- that is `ExternalWritebackService`'s job now (its own + * adapter-agnostic in-memory claim, see that service's module doc + * comment), not this adapter's. This adapter is stateless per call. + * - `baseUrl`/`apiKey`/the Fabric credentials are all constructor + * parameters -- the original hardcoded `process.env.ODOO_BASE_URL`/ + * `process.env.NEXT_PUBLIC_APP_URL`/etc as module-level env reads (v0- + * datahub is a single-tenant app; this repo is a SIAX platform capability + * any customer's tenant can register their own Odoo instance against, the + * same "nothing hardcoded, everything a caller-supplied credential" + * doctrine ../adapters/fabric-onelake-adapter.ts's own header already + * documents for the identical reason). + * - `fetchImpl`/`matchPartner` are injectable (default to the real `fetch` + * / `matchPartnerViaFabric`) purely so tests never make a real network + * call -- the original had no test coverage for this path at all + * (v0-datahub's invoice-bridge.ts has no accompanying test file); this + * port is genuinely tested (see tests/p0/odoo-bills-adapter.test.ts). + */ +import type { WritebackAdapter, WritebackAdapterContext, WritebackRequest, WritebackResult } from "./writeback-adapter"; +import { FabricOneLakeAdapter, type FabricOneLakeConnectionConfig } from "./fabric-onelake-adapter"; + +/** One itemized Vendor Bill line -- same shape as invoice-bridge.ts's `InvoiceForOdoo.lines`. */ +export interface OdooBillLine { + description: string; + quantity: number; + unit_price: number; + vat_rate?: number; + account_bas?: number; +} + +/** + * The structured record this adapter expects as `WritebackRequest.payload` + * -- same field set as invoice-bridge.ts's `InvoiceForOdoo`, plus the guard + * fields (`is_reminder`/`duplicate_flag`/`existing_external_id`) that + * `processInboxToOdoo` read directly off its own `invoice_inbox` row (this + * adapter has no row of its own to read them from, so the caller supplies + * them on the payload instead). + */ +export interface OdooBillPayload { + vendor_name: string; + vendor_org_nr?: string; + /** Pre-resolved Odoo res.partner id -- when absent and `vendor_org_nr` is set, this adapter + * attempts to resolve one via partner matching (see this file's module doc comment). */ + partner_id?: number; + invoice_number: string; + ocr?: string; + amount_gross: number; + amount_excl_vat?: number; + vat?: number; + currency: string; + invoice_date: string; + due_date?: string; + description?: string; + lines?: readonly OdooBillLine[]; + /** Mirrors invoice_inbox.is_reminder -- a payment reminder must never be posted as a new Bill. */ + is_reminder?: boolean; + /** Mirrors invoice_inbox.duplicate_flag -- a detected duplicate must never be posted. */ + duplicate_flag?: boolean; + /** When the caller already knows this record was posted (e.g. re-delivering a webhook), pass + * the known Odoo id here -- this adapter treats it as already-done and returns it unchanged, + * exactly mirroring invoice-bridge.ts's own `invoice.odoo_move_id` early-return. */ + existing_external_id?: string; +} + +export type OdooPartnerMatcher = ( + orgNr: string, + ctx: WritebackAdapterContext, +) => Promise<{ partner_id?: number; partner_name?: string }>; + +export interface OdooBillsAdapterConfig { + /** Base URL of the deployment exposing `/api/odoo/invoices/write` (same route shape as + * v0-datahub's own `/api/odoo/invoices/write` this was ported from). */ + baseUrl: string; + /** Bearer credential for that route. Optional ONLY when `allowUnauthenticated` is explicitly + * set (e.g. a route that authenticates by network policy instead) -- see `isConfigured`. */ + apiKey?: string; + allowUnauthenticated?: boolean; + /** Fabric/OneLake connection used for org-nr -> Odoo partner matching (this file's own + * `matchPartnerViaFabric`), when `matchPartner` is not explicitly supplied. Optional -- + * without it (and without an explicit `matchPartner`), partner matching is skipped entirely + * and the Bill is posted without a partner link, mirroring invoice-bridge.ts's own + * `matchInvoiceToOdooPartner` catch-all ("Ingen partner match — skickar utan + * partner-koppling"). */ + fabric?: FabricOneLakeConnectionConfig; + /** Table name for the partner lookup query -- defaults to `odoo_export__res_partner`, the + * exact table invoice-bridge.ts's own `matchInvoiceToOdooPartner` queried. */ + partnerTable?: string; + /** Injectable partner matcher, for tests or a caller with its own partner-resolution source. + * When supplied, this takes priority over `fabric`. */ + matchPartner?: OdooPartnerMatcher; + /** Injectable `fetch`, so tests never make a real network call. */ + fetchImpl?: typeof fetch; +} + +/** + * Digits-only org-nr -> Odoo partner match, ported 1:1 from invoice-bridge.ts's + * `matchInvoiceToOdooPartner` (see this file's module doc comment). The only + * substitution is the query transport: `FabricOneLakeAdapter.query` instead + * of v0-datahub's `@/lib/warehouse/client` -- same table, same digits-only + * `LIKE` comparison so "556xxx-xxxx"/"SE556xxxxxxxx01"/"5561234567" all + * resolve to the same partner, now parameterized instead of string-interpolated. + */ +export async function matchPartnerViaFabric( + fabric: FabricOneLakeAdapter, + orgNr: string, + partnerTable: string = "odoo_export__res_partner", +): Promise<{ partner_id?: number; partner_name?: string }> { + const digits = orgNr.replace(/[^0-9]/g, ""); + if (!digits) return {}; + try { + const result = await fabric.query<{ id: number; name: string; vat: string }>( + `SELECT TOP 1 id, name, vat FROM ${partnerTable} WHERE REPLACE(REPLACE(vat,'-',''),' ','') LIKE '%' + @digits + '%'`, + { digits }, + ); + if (result.rows.length > 0) { + return { partner_id: result.rows[0]!.id, partner_name: result.rows[0]!.name }; + } + return {}; + } catch { + // Same catch-all as invoice-bridge.ts's own matchInvoiceToOdooPartner: an unreachable/ + // misconfigured warehouse must never fail the whole write -- it degrades to "post without + // a partner link", not a thrown error. + return {}; + } +} + +/** Builds the single synthesized `invoice_line_ids` entry used when `payload.lines` is absent + * -- ported verbatim from invoice-bridge.ts's `processInboxToOdoo` line-synthesis block. */ +function synthesizeSummaryLine(payload: OdooBillPayload): OdooBillLine { + const exclVat = payload.amount_excl_vat ?? payload.amount_gross; + const description = (payload.description || payload.vendor_name || `Faktura ${payload.invoice_number}`).trim(); + return { description, quantity: 1, unit_price: exclVat }; +} + +function isOdooBillPayload(payload: Readonly>): payload is OdooBillPayload & Record { + return ( + typeof payload.vendor_name === "string" && + typeof payload.invoice_number === "string" && + typeof payload.amount_gross === "number" && + typeof payload.currency === "string" && + typeof payload.invoice_date === "string" + ); +} + +export class OdooBillsAdapter implements WritebackAdapter { + readonly adapterId = "odoo_bills"; + private fabricAdapter: FabricOneLakeAdapter | null; + + constructor(private config: OdooBillsAdapterConfig) { + this.fabricAdapter = config.fabric ? new FabricOneLakeAdapter(config.fabric, config.fetchImpl) : null; + } + + isConfigured(): boolean { + return !!this.config.baseUrl && (!!this.config.apiKey || this.config.allowUnauthenticated === true); + } + + async write(request: WritebackRequest, ctx: WritebackAdapterContext): Promise { + if (!this.isConfigured()) { + return { ok: false, error: "odoo_not_configured" }; + } + if (!isOdooBillPayload(request.payload)) { + return { ok: false, error: "invalid_payload_missing_required_fields" }; + } + const payload = request.payload as unknown as OdooBillPayload; + + // Guard 1: reminders/duplicates are never posted -- ported verbatim from + // invoice-bridge.ts's processInboxToOdoo (see this file's module doc comment). + if (payload.is_reminder) return { ok: false, error: "is_reminder" }; + if (payload.duplicate_flag) return { ok: false, error: "duplicate" }; + + // Guard 2: already posted -- idempotent no-op, ported verbatim. + if (payload.existing_external_id) { + return { ok: true, externalId: payload.existing_external_id }; + } + + // Partner match (best-effort, never fails the write) -- ported verbatim in spirit. + let partnerId = payload.partner_id; + if (!partnerId && payload.vendor_org_nr) { + if (this.config.matchPartner) { + const matched = await this.config.matchPartner(payload.vendor_org_nr, ctx); + partnerId = matched.partner_id; + } else if (this.fabricAdapter) { + const matched = await matchPartnerViaFabric(this.fabricAdapter, payload.vendor_org_nr, this.config.partnerTable); + partnerId = matched.partner_id; + } + } + + const lines = payload.lines && payload.lines.length > 0 ? payload.lines : [synthesizeSummaryLine(payload)]; + + const fetchImpl = this.config.fetchImpl ?? fetch; + let res: Response; + try { + res = await fetchImpl(`${this.config.baseUrl}/api/odoo/invoices/write`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(this.config.apiKey ? { Authorization: `Bearer ${this.config.apiKey}` } : {}), + }, + body: JSON.stringify({ + model: "account.move", + invoice: { + partner_id: partnerId, + ref: payload.invoice_number, + invoice_date: payload.invoice_date, + invoice_date_due: payload.due_date, + invoice_line_ids: lines.map((l) => ({ name: l.description, quantity: l.quantity, price_unit: l.unit_price })), + }, + }), + }); + } catch (e) { + return { ok: false, error: `odoo_network_error: ${e instanceof Error ? e.message : String(e)}` }; + } + + if (!res.ok) { + return { ok: false, error: `odoo_http_${res.status}` }; + } + const result: { ok?: boolean; odoo_id?: number; error?: string } = await res.json(); + if (!result.ok || !result.odoo_id) { + return { ok: false, error: result.error ?? "odoo_write_failed" }; + } + return { ok: true, externalId: String(result.odoo_id) }; + } +} diff --git a/src/adapters/writeback-adapter.ts b/src/adapters/writeback-adapter.ts new file mode 100644 index 0000000..b83d571 --- /dev/null +++ b/src/adapters/writeback-adapter.ts @@ -0,0 +1,71 @@ +/** + * src/adapters/writeback-adapter.ts + * =================================== + * The adapter interface `ExternalWritebackService` (../services/ + * external-writeback-service.ts) writes through. One implementation per + * external system (e.g. odoo-bills-adapter.ts's OdooBillsAdapter for Odoo + * Vendor Bills) -- the service itself has zero knowledge of Odoo, Salesforce, + * or any other concrete external system; it only knows "an adapter with this + * adapterId exists, is configured, and can write a record." + */ + +export interface WritebackAdapterContext { + tenant_id: string; + project_id: string; + actor_id: string; + correlation_id: string; +} + +export interface WritebackRequest { + recordType: string; + payload: Readonly>; + /** Same idempotency key the caller gave ExternalWritebackService -- forwarded so an adapter + * with its own persistent storage (unlike the generic service's in-memory claim) can ALSO + * claim defensively at its own layer. See WritebackAdapter.write's own doc comment. */ + idempotencyKey: string; +} + +export interface WritebackResult { + ok: boolean; + /** The external system's own id for the record that was created (or already existed -- + * see WritebackAdapter.write's own doc comment on idempotent re-calls), when ok is true. */ + externalId?: string; + /** A short, stable, machine-readable reason code when ok is false (e.g. "is_reminder", + * "duplicate", "odoo_http_500") -- NOT a free-form human sentence, so callers/tests can + * assert on it reliably. */ + error?: string; +} + +/** + * One external-system write adapter. Registered onto an + * `ExternalWritebackService` via `registerAdapter` under a stable `adapterId` + * (e.g. "odoo_bills") that `ExternalWriteRequest.adapter_id` references. + */ +export interface WritebackAdapter { + readonly adapterId: string; + + /** Whether this adapter has everything it needs (base URL, credentials, etc) to actually + * write -- mirrors DocumentIntelligenceAdapter.isConfigured's "refuse to fabricate a result" + * convention (../adapters/document-intelligence-adapter.ts): an unconfigured adapter must + * never be silently treated as a no-op success. */ + isConfigured(): boolean; + + /** + * Writes one record to the external system. + * + * IDEMPOTENCY, TWO LAYERS (per this task's own instruction not to regress the proven + * idempotent-claim logic ported from v0-datahub's invoice-bridge.ts): + * 1. `ExternalWritebackService` (the caller) already claims `idempotencyKey` in its own + * in-memory map BEFORE ever calling this method, so under normal operation `write` is + * only ever called once per logical record, from one service instance. + * 2. That in-memory claim is process-local and does not survive a restart, nor protect a + * multi-instance deployment. An adapter backed by its own persistent storage (like + * OdooBillsAdapter, when a caller supplies an already-posted record's + * `existing_external_id` in the payload) MUST ALSO refuse to re-post a record that + * already carries an external id -- exactly the "Redan bokförd i Odoo... hoppar över" + * early-return invoice-bridge.ts's own `processInboxToOdoo` performs. This is the + * genuine defense-in-depth the original proven code had; a `WritebackAdapter` + * implementation with real persistent storage should follow the same doctrine. + */ + write(request: WritebackRequest, ctx: WritebackAdapterContext): Promise; +} diff --git a/src/services/document-capture-service.ts b/src/services/document-capture-service.ts new file mode 100644 index 0000000..e73be0e --- /dev/null +++ b/src/services/document-capture-service.ts @@ -0,0 +1,273 @@ +/** + * src/services/document-capture-service.ts + * ========================================= + * Orchestrates the capture half of the 0pt.document_ocr capability: + * poll a customer-registered mailbox (Microsoft Graph, app-only) → + * download PDF attachments → persist the bytes as an artifact → + * enqueue extraction (an N0D job placeholder; a worker or the + * /extraction API route performs the real Document Intelligence call). + * + * Tenant-agnostic: mailbox address + Graph credentials + folder are all + * caller-supplied per invocation — nothing here is stored keyed only by a + * hardcoded mailbox. Nothing here assumes invoices, vendors, or Protosell; + * that logic (vendor-kb matching, dup-hash, DQ scoring, invoice + * classification) was intentionally left behind in + * v0-datahub/lib/invoice-inbox — it does not belong in a generic capture + * pipeline that also has to serve HR forms, contracts, receipts, etc. + * + * KNOWN GAP — ST0RE bytes storage: st0re-client.ts registers artifact + * METADATA only (a pointer to provider/bucket/key); it has no real + * bytes-upload endpoint yet. Until one exists, this service writes + * attachment bytes to LocalBlobStore (src/adapters/local-blob-store.ts), a + * clearly-marked interim local-disk store, then registers that location + * with ST0RE so the artifact/lineage metadata is still correct. Search this + * file for "INTERIM" to find the seam to replace. + * + * Singleton note: unlike source-service.ts / dataset-service.ts (which are + * instantiated fresh per API route file because each request is + * self-contained), DocumentCaptureService holds the only in-memory record + * of *which bytes belong to which document_id* — document-extraction-service + * must read the exact same instance to find them. So this module also + * exports a shared `documentCaptureService` singleton for both API routes + * to import, instead of each route constructing its own instance. + */ +import crypto from "crypto"; +import { GraphMailboxAdapter, type GraphMailboxConfig, type GraphMessage, type GraphAttachment } from "../adapters/graph-mailbox-adapter"; +import { LocalBlobStore } from "../adapters/local-blob-store"; +import { createSt0reClient, type St0reClient } from "../clients/st0re-client"; +import { createN0dClient, type N0dClient } from "../clients/n0d-client"; +import { createAud0Client, type Aud0Client } from "../clients/aud0-client"; +import type { DocumentRecord, DocumentStatus } from "../types/document"; + +export interface CaptureMailboxContext { + tenant_id: string; + project_id: string; + actor_id: string; + correlation_id: string; +} + +export interface CaptureMailboxInput { + mailbox: GraphMailboxConfig; + since_iso?: string; + top?: number; + max_pages?: number; +} + +export interface CaptureMailboxResult { + scanned_messages: number; + captured_documents: DocumentRecord[]; + errors: string[]; +} + +interface StoredBytesLocation { + bucket: string; + key: string; +} + +export class DocumentCaptureService { + private documents: Map = new Map(); + private bytesByDocumentId: Map = new Map(); + private st0re: St0reClient; + private n0d: N0dClient; + private aud0: Aud0Client; + private blobStore: LocalBlobStore; + + constructor(opts?: { st0reBaseUrl?: string; n0dBaseUrl?: string; aud0BaseUrl?: string; blobStore?: LocalBlobStore }) { + this.st0re = createSt0reClient(opts?.st0reBaseUrl); + this.n0d = createN0dClient(opts?.n0dBaseUrl); + this.aud0 = createAud0Client(opts?.aud0BaseUrl); + this.blobStore = opts?.blobStore ?? new LocalBlobStore(); + } + + async captureFromMailbox(input: CaptureMailboxInput, ctx: CaptureMailboxContext): Promise { + if (!ctx.tenant_id || !ctx.project_id || !ctx.actor_id) { + throw new Error("captureFromMailbox: missing tenant_id/project_id/actor_id"); + } + if (!GraphMailboxAdapter.isConfigured(input.mailbox)) { + throw new Error( + "captureFromMailbox: missing mailbox.tenantId/clientId/clientSecret/mailboxAddress", + ); + } + const graph = new GraphMailboxAdapter(input.mailbox); + const result: CaptureMailboxResult = { scanned_messages: 0, captured_documents: [], errors: [] }; + + const messages = await graph.pollMessages({ + sinceISO: input.since_iso, + top: input.top, + maxPages: input.max_pages, + }); + + for (const message of messages) { + result.scanned_messages++; + if (!message.hasAttachments) continue; + try { + const attachments = await graph.listAttachments(message.id); + for (const attachment of attachments) { + if (!attachment.isPdf) continue; + const document = await this.captureAttachment(input.mailbox, message, attachment, ctx); + result.captured_documents.push(document); + } + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + result.errors.push(`${message.id}: ${msg.slice(0, 200)}`); + } + } + return result; + } + + private async captureAttachment( + mailbox: GraphMailboxConfig, + message: GraphMessage, + attachment: GraphAttachment, + ctx: CaptureMailboxContext, + ): Promise { + const documentId = `doc_${crypto.randomUUID()}`; + const now = new Date().toISOString(); + const bucket = "0pt-documents"; + const safeName = (attachment.name || "attachment.pdf").replace(/[^A-Za-z0-9._-]/g, "_"); + const key = `${ctx.tenant_id}/${ctx.project_id}/${message.id}/${attachment.id}/${safeName}`; + + // INTERIM — see file header. Real bytes persistence until ST0RE exposes an upload endpoint. + const written = await this.blobStore.write(bucket, key, attachment.contentBytes); + const checksum = crypto.createHash("sha256").update(attachment.contentBytes).digest("hex"); + + let record: DocumentRecord = { + document_id: documentId, + tenant_id: ctx.tenant_id, + project_id: ctx.project_id, + mailbox_address: mailbox.mailboxAddress, + graph_message_id: message.id, + graph_attachment_id: attachment.id, + filename: attachment.name ?? undefined, + content_type: attachment.contentType ?? undefined, + size_bytes: attachment.sizeBytes ?? attachment.contentBytes.length, + received_at: message.receivedAt ?? undefined, + sender_email: message.senderEmail ?? undefined, + sender_name: message.senderName ?? undefined, + subject: message.subject ?? undefined, + status: "captured", + correlation_id: ctx.correlation_id, + created_at: now, + }; + this.documents.set(documentId, record); + this.bytesByDocumentId.set(documentId, { bucket: written.bucket, key: written.key }); + + // Register the artifact pointer with ST0RE (metadata only — see file header gap note). + const artifact = await this.st0re.createArtifact({ + tenant_id: ctx.tenant_id, + project_id: ctx.project_id, + artifact_type: "document", + app_id: "0pt", + produced_by: "0pt.capture_document", + object_location: { provider: written.provider, bucket: written.bucket, key: written.key, region: "local" }, + content_type: attachment.contentType || "application/pdf", + size_bytes: written.sizeBytes, + checksum_sha256: checksum, + pii_level: "medium", + retention_policy: "document_capture_standard", + data_residency: "eu", + risk_level: "L3", + correlation_id: ctx.correlation_id, + }); + + // Enqueue extraction. A worker (out of scope here) or a caller hitting the + // /documents/[documentId]/extraction route performs the real DI call. + const job = await this.n0d.createJob({ + tenant_id: ctx.tenant_id, + project_id: ctx.project_id, + job_type: "0pt.document_extraction_job", + capability_id: "0pt.extract_document", + requested_by_app_id: "0pt", + requested_by_actor_id: ctx.actor_id, + input_artifact_ids: [artifact.artifact_id], + output_artifact_type: "document_extraction", + requires_backbone: true, + requires_gpu: false, + required_node_roles: ["compute_general"], + resource_profile: "cpu_io_medium", + risk_level: "L3", + correlation_id: ctx.correlation_id, + }); + + record = { + ...record, + st0re_artifact_id: artifact.artifact_id, + n0d_job_id: job.job_id, + status: "extraction_requested", + updated_at: new Date().toISOString(), + }; + this.documents.set(documentId, record); + + await this.aud0.ingestEvent({ + event_id: `evt_0pt_document_captured_${documentId}`, + event_type: "0pt.document_captured", + tenant_id: ctx.tenant_id, + project_id: ctx.project_id, + actor_id: ctx.actor_id, + app_id: "0pt", + capability_id: "0pt.capture_document", + resource_type: "document", + resource_id: documentId, + risk_level: "L3", + payload: { + mailbox_address: mailbox.mailboxAddress, + graph_message_id: message.id, + filename: attachment.name, + size_bytes: written.sizeBytes, + st0re_artifact_id: artifact.artifact_id, + n0d_job_id: job.job_id, + }, + created_at: new Date().toISOString(), + correlation_id: ctx.correlation_id, + }); + + await this.aud0.ingestLineageEvent({ + lineage_event_id: `lineage_0pt_document_captured_${documentId}`, + tenant_id: ctx.tenant_id, + project_id: ctx.project_id, + app_id: "0pt", + capability_id: "0pt.capture_document", + source_type: "mailbox", + source_id: mailbox.mailboxAddress, + target_type: "document", + target_id: documentId, + lineage_type: "produced", + metadata: { graph_message_id: message.id, st0re_artifact_id: artifact.artifact_id }, + created_at: new Date().toISOString(), + correlation_id: ctx.correlation_id, + }); + + return record; + } + + async getDocument(documentId: string): Promise { + const record = this.documents.get(documentId); + if (!record) throw new Error(`Document ${documentId} not found`); + return record; + } + + async listDocuments(filter?: { tenant_id?: string; project_id?: string; status?: DocumentStatus }): Promise { + let results = Array.from(this.documents.values()); + if (filter?.tenant_id) results = results.filter((r) => r.tenant_id === filter.tenant_id); + if (filter?.project_id) results = results.filter((r) => r.project_id === filter.project_id); + if (filter?.status) results = results.filter((r) => r.status === filter.status); + return results; + } + + /** Read back the raw bytes captured for a document. INTERIM local store — see file header. */ + async getDocumentBytes(documentId: string): Promise { + const loc = this.bytesByDocumentId.get(documentId); + if (!loc) throw new Error(`Document ${documentId} has no stored bytes`); + return this.blobStore.read(loc.bucket, loc.key); + } + + setStatus(documentId: string, status: DocumentStatus): void { + const record = this.documents.get(documentId); + if (!record) throw new Error(`Document ${documentId} not found`); + record.status = status; + record.updated_at = new Date().toISOString(); + } +} + +/** Shared singleton — see the "Singleton note" in the file header for why. */ +export const documentCaptureService = new DocumentCaptureService(); diff --git a/src/services/document-extraction-service.ts b/src/services/document-extraction-service.ts new file mode 100644 index 0000000..003bcd0 --- /dev/null +++ b/src/services/document-extraction-service.ts @@ -0,0 +1,267 @@ +/** + * src/services/document-extraction-service.ts + * ============================================== + * Evidence-based field extraction for a captured document. + * + * Calls DocumentIntelligenceAdapter and NEVER fabricates a value: a field + * key is only populated with a non-null value when Document Intelligence + * actually returned one, and it always carries DI's own confidence score + * (or null when DI reported none). Fields DI did not find are left blank — + * { value: null, confidence: , found: false } — + * never defaulted, guessed, or backfilled from anything else. This mirrors + * the evidence-based discipline in + * v0-datahub/lib/invoice-inbox/doc-intelligence.ts exactly (it never + * invented an amount/vendor/date that DI didn't return); it is a + * correctness property, not a style preference, so it is enforced twice — + * once inside the adapter's field normalizer, and again defensively here. + * + * modelId is caller-supplied ("prebuilt-invoice", "prebuilt-receipt", + * "prebuilt-layout", or a custom model id) — this service does not assume + * any particular document type. Invoice-specific reasoning (vendor + * matching, dup-hash, DQ scoring, invoice classification) was intentionally + * left in v0-datahub/lib/invoice-inbox and is NOT ported here. + */ +import crypto from "crypto"; +import { + DocumentIntelligenceAdapter, + type DocumentIntelligenceConfig, + type ExtractedField as AdapterField, +} from "../adapters/document-intelligence-adapter"; +import { LocalBlobStore } from "../adapters/local-blob-store"; +import { createCl0udClient, type Cl0udClient } from "../clients/cl0ud-client"; +import { createSt0reClient, type St0reClient } from "../clients/st0re-client"; +import { createAud0Client, type Aud0Client } from "../clients/aud0-client"; +import { DocumentCaptureService, documentCaptureService } from "./document-capture-service"; +import type { DocumentExtractionRecord, ExtractedField } from "../types/document-extraction"; + +export interface ExtractDocumentContext { + tenant_id: string; + project_id: string; + actor_id: string; + correlation_id: string; + /** e.g. "prebuilt-invoice" | "prebuilt-receipt" | "prebuilt-layout" | a custom model id. */ + model_id: string; +} + +/** + * Defensive re-application of the "never fabricate" rule at the service + * boundary. Even though the adapter's normalizeField() already enforces + * this, a field is only ever considered found here if it is BOTH marked + * found by the adapter AND carries a genuinely non-empty value — so a + * malformed/partial adapter response can never leak a fabricated value + * through this service. + */ +function toEvidenceFields(fields: Record): Record { + const out: Record = {}; + for (const [key, f] of Object.entries(fields)) { + const found = !!f?.found && f.value !== null && f.value !== undefined && f.value !== ""; + out[key] = { value: found ? f.value : null, confidence: f?.confidence ?? null, found }; + } + return out; +} + +export class DocumentExtractionService { + private extractions: Map = new Map(); + private cl0ud: Cl0udClient; + private st0re: St0reClient; + private aud0: Aud0Client; + private blobStore: LocalBlobStore; + private di: DocumentIntelligenceAdapter | null; + + constructor( + private captureService: DocumentCaptureService, + opts?: { + cl0udBaseUrl?: string; + st0reBaseUrl?: string; + aud0BaseUrl?: string; + documentIntelligence?: DocumentIntelligenceConfig; + blobStore?: LocalBlobStore; + }, + ) { + this.cl0ud = createCl0udClient(opts?.cl0udBaseUrl); + this.st0re = createSt0reClient(opts?.st0reBaseUrl); + this.aud0 = createAud0Client(opts?.aud0BaseUrl); + this.blobStore = opts?.blobStore ?? new LocalBlobStore(); + this.di = + opts?.documentIntelligence && DocumentIntelligenceAdapter.isConfigured(opts.documentIntelligence) + ? new DocumentIntelligenceAdapter(opts.documentIntelligence) + : null; + } + + async extractDocument(documentId: string, ctx: ExtractDocumentContext): Promise { + if (!this.di) { + throw new Error( + "DocumentExtractionService: Document Intelligence is not configured (missing endpoint/apiKey) — refusing to fabricate a result", + ); + } + if (!ctx.model_id) { + throw new Error("extractDocument: missing model_id"); + } + const document = await this.captureService.getDocument(documentId); + + const policy = await this.cl0ud.evaluatePolicy({ + tenant_id: ctx.tenant_id, + project_id: ctx.project_id, + actor_id: ctx.actor_id, + capability_id: "0pt.extract_document", + resource_type: "document", + resource_id: documentId, + risk_level: "L3", + correlation_id: ctx.correlation_id, + }); + if (policy.decision === "deny") throw new Error(`Policy denied: ${policy.reason}`); + if (policy.decision === "approval_required") throw new Error(`Policy requires approval: ${policy.reason}`); + + const extractionId = `dex_${crypto.randomUUID()}`; + const now = new Date().toISOString(); + let record: DocumentExtractionRecord = { + extraction_id: extractionId, + document_id: documentId, + tenant_id: ctx.tenant_id, + project_id: ctx.project_id, + model_id: ctx.model_id, + status: "requested", + fields: {}, + line_items: [], + fields_found_count: 0, + fields_total_count: 0, + average_confidence: null, + document_confidence: null, + correlation_id: ctx.correlation_id, + created_at: now, + }; + this.extractions.set(extractionId, record); + + try { + const bytes = await this.captureService.getDocumentBytes(documentId); + const diResult = await this.di.analyzeDocument(bytes, ctx.model_id); + + const fields = toEvidenceFields(diResult.fields); + const lineItems = Object.values(diResult.arrayFields).flatMap((items) => items.map((item) => toEvidenceFields(item))); + + const foundCount = Object.values(fields).filter((f) => f.found).length; + // Average only over FOUND fields — a not-found field's confidence (e.g. "31% sure this + // region isn't a date after all") describes DI's certainty about absence, not a value, + // and must not drag down the confidence of the values that were actually extracted. + const confidences = Object.values(fields) + .filter((f) => f.found) + .map((f) => f.confidence) + .filter((c): c is number => c != null); + const averageConfidence = confidences.length ? confidences.reduce((a, b) => a + b, 0) / confidences.length : null; + + // Persist the extraction result as an artifact too — same interim-storage gap as + // document-capture-service.ts (see that file's header for the full explanation). + const payloadJson = Buffer.from( + JSON.stringify({ model_id: ctx.model_id, fields, line_items: lineItems, content: diResult.content }), + "utf8", + ); + const bucket = "0pt-document-extractions"; + const key = `${ctx.tenant_id}/${ctx.project_id}/${documentId}/${extractionId}.json`; + const written = await this.blobStore.write(bucket, key, payloadJson); + const artifact = await this.st0re.createArtifact({ + tenant_id: ctx.tenant_id, + project_id: ctx.project_id, + artifact_type: "document_extraction", + app_id: "0pt", + produced_by: "0pt.extract_document", + object_location: { provider: written.provider, bucket: written.bucket, key: written.key, region: "local" }, + content_type: "application/json", + size_bytes: written.sizeBytes, + checksum_sha256: crypto.createHash("sha256").update(payloadJson).digest("hex"), + pii_level: "medium", + retention_policy: "document_extraction_standard", + data_residency: "eu", + risk_level: "L3", + correlation_id: ctx.correlation_id, + }); + + record = { + ...record, + status: "completed", + fields, + line_items: lineItems, + fields_found_count: foundCount, + fields_total_count: Object.keys(fields).length, + average_confidence: averageConfidence, + document_confidence: diResult.documentConfidence, + st0re_artifact_id: artifact.artifact_id, + updated_at: new Date().toISOString(), + }; + this.extractions.set(extractionId, record); + this.captureService.setStatus(documentId, "extracted"); + + await this.aud0.ingestEvent({ + event_id: `evt_0pt_document_extracted_${extractionId}`, + event_type: "0pt.document_extracted", + tenant_id: ctx.tenant_id, + project_id: ctx.project_id, + actor_id: ctx.actor_id, + app_id: "0pt", + capability_id: "0pt.extract_document", + resource_type: "document_extraction", + resource_id: extractionId, + risk_level: "L3", + payload: { + document_id: documentId, + model_id: ctx.model_id, + fields_found_count: foundCount, + fields_total_count: Object.keys(fields).length, + average_confidence: averageConfidence, + st0re_artifact_id: artifact.artifact_id, + }, + created_at: new Date().toISOString(), + correlation_id: ctx.correlation_id, + }); + + await this.aud0.ingestLineageEvent({ + lineage_event_id: `lineage_0pt_document_extracted_${extractionId}`, + tenant_id: ctx.tenant_id, + project_id: ctx.project_id, + app_id: "0pt", + capability_id: "0pt.extract_document", + source_type: "document", + source_id: documentId, + target_type: "document_extraction", + target_id: extractionId, + lineage_type: "derived_from", + metadata: { model_id: ctx.model_id, st0re_artifact_id: artifact.artifact_id }, + created_at: new Date().toISOString(), + correlation_id: ctx.correlation_id, + }); + + return record; + } catch (e: unknown) { + const message = e instanceof Error ? e.message : String(e); + record = { ...record, status: "failed", error: message, updated_at: new Date().toISOString() }; + this.extractions.set(extractionId, record); + try { + this.captureService.setStatus(documentId, "extraction_failed"); + } catch { + /* document may already be gone in edge-case tests — extraction failure record still stands */ + } + throw e; + } + } + + async getExtraction(extractionId: string): Promise { + const record = this.extractions.get(extractionId); + if (!record) throw new Error(`Document extraction ${extractionId} not found`); + return record; + } + + async listExtractionsForDocument(documentId: string): Promise { + return Array.from(this.extractions.values()).filter((r) => r.document_id === documentId); + } +} + +function buildDocumentIntelligenceConfigFromEnv(): DocumentIntelligenceConfig | undefined { + const endpoint = process.env.DOC_INTELLIGENCE_ENDPOINT; + const apiKey = process.env.DOC_INTELLIGENCE_KEY; + if (!endpoint || !apiKey) return undefined; + return { endpoint, apiKey, apiVersion: process.env.DOC_INTELLIGENCE_API_VERSION }; +} + +/** Shared singleton (see document-capture-service.ts "Singleton note") wired to the shared capture service. */ +export const documentExtractionService = new DocumentExtractionService(documentCaptureService, { + documentIntelligence: buildDocumentIntelligenceConfigFromEnv(), +}); diff --git a/src/services/external-writeback-service.ts b/src/services/external-writeback-service.ts new file mode 100644 index 0000000..e8c893d --- /dev/null +++ b/src/services/external-writeback-service.ts @@ -0,0 +1,258 @@ +/** + * src/services/external-writeback-service.ts + * ============================================= + * `0pt.write_external_record` -- "POST a validated, already-approved + * structured record to a registered external-system adapter." Generic in its + * core: this service has no Odoo/invoice/Protosell-specific logic anywhere + * in it -- it resolves an `adapter_id` to a registered `WritebackAdapter` + * (../adapters/writeback-adapter.ts) and delegates the actual write to it. + * `../adapters/odoo-bills-adapter.ts`'s OdooBillsAdapter is ONE such + * adapter, registered by a caller the same way any other adapter would be -- + * this service never imports it directly. + * + * WHY THIS EXISTS / WHERE IT SITS IN THE CHAIN: `act0/src/ + * approval-writeback-templates.ts`'s generic `0pt_document_approval_then_ + * external_write` Template chains a captured+extracted document + * (0pt.document_ocr's real terminal event, `0pt.document_extracted`) through + * an Approval Engine gate, THEN calls this capability. By the time + * `writeExternalRecord` runs, a human has already approved the write -- this + * service still re-checks CL0UD policy (per docs/PRODUCT_PLAN.md section + * 4.1's hard rule that CL0UD is always re-consulted, the same rule + * quality-check-service.ts/document-extraction-service.ts already follow), + * but does not itself re-verify the upstream Approval record (ACT0 owns + * approvals; 0pt's own capability boundary is "did CL0UD say yes for THIS + * call", same separation of concerns every other 0pt service in this repo + * already draws). + * + * IDEMPOTENCY (per this task's own instruction not to regress the proven + * pattern ported from v0-datahub's `lib/odoo/invoice-bridge.ts` + * `processInboxToOdoo`): that original code CLAIMS a row (an atomic + * Postgres `UPDATE ... WHERE odoo_move_id IS NULL AND odoo_state != 'posting'`) + * BEFORE ever calling the external system, so two concurrent triggers (a + * manual approve racing a cron, or a retried ACT0 step racing itself) cannot + * both post the same record. This service ports that SAME "claim before you + * call out" shape forward as a generic, adapter-agnostic in-memory claim, + * keyed on `(adapter_id, idempotency_key)`: + * - the claim check-and-set happens SYNCHRONOUSLY, before this function's + * first `await` -- JavaScript's single-threaded run-to-first-await + * semantics make this exactly as race-free, within one process, as the + * original's atomic SQL UPDATE was across concurrent DB clients (see + * `writeExternalRecord`'s own inline comment for exactly where the line + * is drawn, and tests/p0/external-writeback-service.test.ts's own + * concurrency test, which calls `writeExternalRecord` twice with the + * same idempotency key via `Promise.all` and asserts the underlying + * adapter's `write` is invoked exactly once). + * - HONEST SCOPE: this in-memory claim is process-local -- it does not + * survive a restart and does not protect a multi-instance deployment, + * unlike the original's real Postgres row lock. An adapter with its own + * persistent storage (OdooBillsAdapter) additionally refuses to + * re-post a record whose payload already carries an `existing_external_id` + * -- see WritebackAdapter.write's own doc comment for why this + * second layer matters and odoo-bills-adapter.ts for exactly how it is + * ported. A production deployment backing this service with real + * persistent storage (same "pure core, thin adapter" doctrine every + * other 0pt service in this repo already documents) should replace this + * in-memory Map with a real claim row, at which point the SAME + * synchronous-claim-before-any-await shape still applies, now backed by + * a real transaction instead of a JS Map. + */ +import crypto from "crypto"; +import { createCl0udClient, type Cl0udClient } from "../clients/cl0ud-client"; +import { createAud0Client, type Aud0Client } from "../clients/aud0-client"; +import type { WritebackAdapter } from "../adapters/writeback-adapter"; +import type { ExternalWriteRecord, ExternalWriteRequest } from "../types/external-writeback"; + +export interface ExternalWritebackServiceOptions { + cl0udBaseUrl?: string; + aud0BaseUrl?: string; + adapters?: readonly WritebackAdapter[]; +} + +export class ExternalWritebackService { + private writes: Map = new Map(); + /** `${adapter_id}::${idempotency_key}` -> write_id. See this module's own doc comment's + * "IDEMPOTENCY" section for exactly what this does and does not guarantee. */ + private claimedByIdempotencyKey: Map = new Map(); + private adapters: Map = new Map(); + private cl0ud: Cl0udClient; + private aud0: Aud0Client; + + constructor(opts?: ExternalWritebackServiceOptions) { + this.cl0ud = createCl0udClient(opts?.cl0udBaseUrl); + this.aud0 = createAud0Client(opts?.aud0BaseUrl); + for (const adapter of opts?.adapters ?? []) { + this.registerAdapter(adapter); + } + } + + registerAdapter(adapter: WritebackAdapter): void { + this.adapters.set(adapter.adapterId, adapter); + } + + getAdapter(adapterId: string): WritebackAdapter { + const adapter = this.adapters.get(adapterId); + if (!adapter) throw new Error(`ExternalWritebackService: no adapter registered for '${adapterId}'`); + return adapter; + } + + async writeExternalRecord(request: ExternalWriteRequest): Promise { + const adapter = this.adapters.get(request.adapter_id); + if (!adapter) { + throw new Error(`ExternalWritebackService: no adapter registered for '${request.adapter_id}'`); + } + if (!adapter.isConfigured()) { + throw new Error( + `ExternalWritebackService: adapter '${request.adapter_id}' is not configured — refusing to fabricate a write`, + ); + } + + // --- SYNCHRONOUS claim section: no `await` above this line, and none until the claim is + // set below. This is what makes the claim race-free under this module's own doc comment's + // "IDEMPOTENCY" doctrine -- see that comment for the full explanation and its own scope + // caveat (process-local only). + const claimKey = `${request.adapter_id}::${request.idempotency_key}`; + const existingWriteId = this.claimedByIdempotencyKey.get(claimKey); + if (existingWriteId) { + const existing = this.writes.get(existingWriteId); + if (existing && existing.status === "written") { + return existing; // already posted -- idempotent no-op, matches invoice-bridge.ts's own + // "Redan bokförd i Odoo... hoppar över" early return. + } + if (existing && existing.status === "claimed") { + // A write for this exact idempotency key is already in flight (or, in this in-memory + // reference implementation, was left 'claimed' by a caller that never awaited the + // result) -- do not race a second POST to the external system. Matches + // invoice-bridge.ts's own "Redan under postning eller bokförd — hoppar över" claim- + // conflict response. + return { ...existing, status: "skipped_duplicate" }; + } + // existing.status === 'failed' (or the record is missing for any other reason): fall + // through and re-claim -- a failed write is safe and expected to retry, matching + // invoice-bridge.ts's own "Release the 'posting' claim... can be retried later" behavior. + } + + const writeId = `ewr_${crypto.randomUUID()}`; + const now = new Date().toISOString(); + let record: ExternalWriteRecord = { + write_id: writeId, + adapter_id: request.adapter_id, + idempotency_key: request.idempotency_key, + record_type: request.record_type, + status: "claimed", + tenant_id: request.tenant_id, + project_id: request.project_id, + actor_id: request.actor_id, + correlation_id: request.correlation_id, + created_at: now, + }; + this.writes.set(writeId, record); + this.claimedByIdempotencyKey.set(claimKey, writeId); + // --- End synchronous claim section. Every branch below is async and safe to interleave. --- + + try { + const policy = await this.cl0ud.evaluatePolicy({ + tenant_id: request.tenant_id, + project_id: request.project_id, + actor_id: request.actor_id, + capability_id: "0pt.write_external_record", + resource_type: "external_write", + resource_id: writeId, + risk_level: "L4", + correlation_id: request.correlation_id, + }); + if (policy.decision === "deny") { + throw new Error(`Policy denied: ${policy.reason}`); + } + if (policy.decision === "approval_required") { + // Per this service's own module doc comment: the ACT0 Approval Engine gate that + // precedes this capability in the approval-writeback-templates.ts chain is a SEPARATE + // mechanism from CL0UD's own policy decision here -- if CL0UD independently still says + // approval_required for this specific call, this service does not silently bypass it + // (same "never lie about what happened" doctrine every other 0pt service in this repo + // follows for a denied/approval_required decision). + throw new Error(`Policy requires approval: ${policy.reason}`); + } + + const result = await adapter.write( + { recordType: request.record_type, payload: request.payload, idempotencyKey: request.idempotency_key }, + { + tenant_id: request.tenant_id, + project_id: request.project_id, + actor_id: request.actor_id, + correlation_id: request.correlation_id, + }, + ); + if (!result.ok) { + throw new Error(result.error ?? "adapter_write_failed"); + } + + record = { ...record, status: "written", external_id: result.externalId, updated_at: new Date().toISOString() }; + this.writes.set(writeId, record); + + await this.aud0.ingestEvent({ + event_id: `evt_0pt_external_record_written_${writeId}`, + event_type: "0pt.external_record_written", + tenant_id: request.tenant_id, + project_id: request.project_id, + actor_id: request.actor_id, + app_id: "0pt", + capability_id: "0pt.write_external_record", + resource_type: "external_write", + resource_id: writeId, + risk_level: "L4", + payload: { + adapter_id: request.adapter_id, + record_type: request.record_type, + idempotency_key: request.idempotency_key, + external_id: result.externalId, + }, + created_at: new Date().toISOString(), + correlation_id: request.correlation_id, + }); + + await this.aud0.ingestLineageEvent({ + lineage_event_id: `lineage_0pt_external_write_${writeId}`, + tenant_id: request.tenant_id, + project_id: request.project_id, + app_id: "0pt", + capability_id: "0pt.write_external_record", + // `idempotency_key` is, by convention (not enforced by this generic service), the + // upstream record's stable id -- e.g. a document_extraction_id when this write was + // triggered by act0's document_approval_then_external_write template. + source_type: "document_extraction", + source_id: request.idempotency_key, + target_type: "external_record", + target_id: result.externalId ?? writeId, + lineage_type: "produced", + metadata: { adapter_id: request.adapter_id, record_type: request.record_type }, + created_at: new Date().toISOString(), + correlation_id: request.correlation_id, + }); + + return record; + } catch (e: unknown) { + const message = e instanceof Error ? e.message : String(e); + record = { ...record, status: "failed", error: message, updated_at: new Date().toISOString() }; + this.writes.set(writeId, record); + throw e; + } + } + + async getWrite(writeId: string): Promise { + const record = this.writes.get(writeId); + if (!record) throw new Error(`External write ${writeId} not found`); + return record; + } + + async findByIdempotencyKey(adapterId: string, idempotencyKey: string): Promise { + const writeId = this.claimedByIdempotencyKey.get(`${adapterId}::${idempotencyKey}`); + return writeId ? this.writes.get(writeId) : undefined; + } +} + +/** Shared singleton, same "cross-route in-memory state" convention document-capture-service.ts's + * own header documents (this service's writes/claims must be visible across whatever API routes + * call writeExternalRecord vs getWrite) -- no adapters pre-registered; a caller (an API route, + * a test, or a future 0pt/api/documents/writeback route) registers whichever adapters it needs + * via registerAdapter before calling writeExternalRecord. */ +export const externalWritebackService = new ExternalWritebackService(); diff --git a/src/services/source-query-service.ts b/src/services/source-query-service.ts new file mode 100644 index 0000000..4746963 --- /dev/null +++ b/src/services/source-query-service.ts @@ -0,0 +1,226 @@ +/** + * src/services/source-query-service.ts + * ======================================= + * 0pt.query_source — bounded, timeout-guarded synchronous query execution + * against a registered Source (source_type = "database", + * adapter_id = "fabric_onelake_sql" today; extensible via `ADAPTER_REGISTRY`). + * + * This is deliberately NOT a general-purpose query engine: it enforces a + * max-rows and max-time ceiling on every call and is meant for small, + * interactive/preview-style reads. Anything heavier (large exports, long + * scans, scheduled pulls) must go through N0D as a job — see + * docs/P0_QUERY_SOURCE_FLOW.md — the same "synchronous vs. N0D job" split + * `quality-check-service.ts` uses for quality checks. + * + * Reuses the existing SourceService (src/services/source-service.ts) purely + * through its public `getSource()` — this file does not modify that class or + * the Source/CreateSourceInput types, which already carry the `adapter_id` + * field this capability depends on. + */ + +import type { Source } from "../types/source"; +import { SourceService } from "./source-service"; +import { createAud0Client, type Aud0Client } from "../clients/aud0-client"; +import { FabricOneLakeAdapter, type FabricOneLakeConnectionConfig, type AdapterQueryResult } from "../adapters/fabric-onelake-adapter"; + +/** Default bound applied when the caller does not specify one. */ +export const SOURCE_QUERY_DEFAULT_MAX_ROWS = 1000; +/** Absolute ceiling — a caller cannot raise max_rows above this on the synchronous path. */ +export const SOURCE_QUERY_HARD_MAX_ROWS = 10_000; +/** Default timeout applied when the caller does not specify one. */ +export const SOURCE_QUERY_DEFAULT_TIMEOUT_MS = 10_000; +/** Absolute ceiling — a caller cannot raise timeout_ms above this on the synchronous path. */ +export const SOURCE_QUERY_HARD_MAX_TIMEOUT_MS = 30_000; + +/** Adapter ids supported by the synchronous query path today. */ +export type SupportedQueryAdapterId = "fabric_onelake_sql"; + +export class SourceQueryTimeoutError extends Error {} +export class SourceQueryUnsupportedError extends Error {} + +export interface QuerySourceInput { + source_id: string; + query: string; + params?: Record; + tenant_id: string; + project_id: string; + actor_id?: string; + correlation_id?: string; + /** Clamped to [1, SOURCE_QUERY_HARD_MAX_ROWS]. Defaults to SOURCE_QUERY_DEFAULT_MAX_ROWS. */ + max_rows?: number; + /** Clamped to [1, SOURCE_QUERY_HARD_MAX_TIMEOUT_MS]. Defaults to SOURCE_QUERY_DEFAULT_TIMEOUT_MS. */ + timeout_ms?: number; + /** + * Connection config for the source's adapter. Never persisted by this + * service and never defaulted from an env var — tenant-agnostic by + * construction, same as the adapter itself. Today only the + * "fabric_onelake_sql" shape is supported. + */ + connection_config: FabricOneLakeConnectionConfig; +} + +export interface QuerySourceResult { + source_id: string; + rows: Record[]; + row_count: number; + truncated: boolean; + duration_ms: number; + bounds: { max_rows: number; timeout_ms: number }; +} + +function clamp(value: number, min: number, max: number): number { + if (!Number.isFinite(value)) return min; + return Math.min(Math.max(value, min), max); +} + +export class SourceQueryService { + private sourceService: SourceService; + private aud0: Aud0Client; + + constructor(opts?: { sourceService?: SourceService; aud0BaseUrl?: string }) { + // Accepts an existing SourceService so callers (e.g. the API route) share + // one in-memory registry instead of each constructing an empty one. + this.sourceService = opts?.sourceService ?? new SourceService(); + this.aud0 = createAud0Client(opts?.aud0BaseUrl); + } + + async querySource(input: QuerySourceInput): Promise { + const startedAt = Date.now(); + const source = await this.getQueryableSource(input.source_id); + + const maxRows = clamp(input.max_rows ?? SOURCE_QUERY_DEFAULT_MAX_ROWS, 1, SOURCE_QUERY_HARD_MAX_ROWS); + const timeoutMs = clamp(input.timeout_ms ?? SOURCE_QUERY_DEFAULT_TIMEOUT_MS, 1, SOURCE_QUERY_HARD_MAX_TIMEOUT_MS); + const correlationId = input.correlation_id ?? `corr_0pt_query_source_${input.source_id}_${startedAt}`; + + const adapter = this.buildAdapter(source, input.connection_config); + try { + const { rows, durationMs } = await this.runWithTimeout(adapter.query(input.query, input.params), timeoutMs); + + const truncated = rows.length > maxRows; + const boundedRows = truncated ? rows.slice(0, maxRows) : rows; + + await this.emitQueriedEvent({ + source, + input, + correlationId, + rowCount: boundedRows.length, + truncated, + durationMs, + ok: true, + }); + + return { + source_id: source.source_id, + rows: boundedRows, + row_count: boundedRows.length, + truncated, + duration_ms: durationMs, + bounds: { max_rows: maxRows, timeout_ms: timeoutMs }, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + await this.emitQueriedEvent({ + source, + input, + correlationId, + rowCount: 0, + truncated: false, + durationMs: Date.now() - startedAt, + ok: false, + error: message, + }); + throw err instanceof Error ? err : new Error(message); + } finally { + // Each call opens its own short-lived adapter/pool (see buildAdapter) — + // always release it, whether the query succeeded, failed, or timed out. + await adapter.close().catch(() => {}); + } + } + + private async getQueryableSource(sourceId: string): Promise { + const source = await this.sourceService.getSource(sourceId); + if (source.source_type !== "database") { + throw new SourceQueryUnsupportedError( + `Source ${sourceId} has source_type "${source.source_type}", not "database" — the synchronous 0pt.query_source path only ` + + `supports database sources. Read non-database sources through a N0D job instead.`, + ); + } + if (source.adapter_id !== "fabric_onelake_sql") { + throw new SourceQueryUnsupportedError( + `Source ${sourceId} has adapter_id "${source.adapter_id ?? "(none)"}" — the synchronous 0pt.query_source path currently only ` + + `supports adapter_id "fabric_onelake_sql". Register the source with that adapter_id, or run the read through N0D as a job instead.`, + ); + } + return source; + } + + private buildAdapter(source: Source, connectionConfig: FabricOneLakeConnectionConfig): FabricOneLakeAdapter { + if (!connectionConfig) { + throw new Error(`Source ${source.source_id}: connection_config is required to query a fabric_onelake_sql source`); + } + return new FabricOneLakeAdapter(connectionConfig); + } + + /** + * Races the adapter query against a timeout. Note this bounds *waiting* + * time on the caller's side — it does not cancel an in-flight TDS request + * on timeout (the `mssql` driver has no cooperative cancellation hook used + * here), so a timed-out query may still complete against the warehouse + * after this call has already thrown. That is an accepted trade-off for a + * synchronous, best-effort preview path; anything where that matters + * belongs on the N0D job path instead. + */ + private async runWithTimeout(promise: Promise>, timeoutMs: number): Promise> { + let timer: ReturnType | undefined; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => { + reject( + new SourceQueryTimeoutError( + `source query exceeded the bounded synchronous timeout of ${timeoutMs}ms — run this query through N0D as a job instead of the synchronous 0pt.query_source path`, + ), + ); + }, timeoutMs); + }); + try { + return await Promise.race([promise, timeout]); + } finally { + clearTimeout(timer); + } + } + + private async emitQueriedEvent(args: { + source: Source; + input: QuerySourceInput; + correlationId: string; + rowCount: number; + truncated: boolean; + durationMs: number; + ok: boolean; + error?: string; + }): Promise { + const { source, input, correlationId, rowCount, truncated, durationMs, ok, error } = args; + await this.aud0.ingestEvent({ + event_id: `evt_0pt_source_queried_${source.source_id}_${Date.now()}`, + event_type: "0pt.source_queried", + tenant_id: input.tenant_id, + project_id: input.project_id, + actor_id: input.actor_id ?? "system", + app_id: "0pt", + capability_id: "0pt.query_source", + resource_type: "source", + resource_id: source.source_id, + risk_level: "L4", + payload: { + adapter_id: source.adapter_id, + ok, + row_count: rowCount, + truncated, + duration_ms: durationMs, + query_preview: input.query.slice(0, 200), + ...(error ? { error } : {}), + }, + created_at: new Date().toISOString(), + correlation_id: correlationId, + }); + } +} diff --git a/src/types/document-extraction.ts b/src/types/document-extraction.ts new file mode 100644 index 0000000..72b816c --- /dev/null +++ b/src/types/document-extraction.ts @@ -0,0 +1,27 @@ +import type { ExtractedField } from "../adapters/document-intelligence-adapter"; + +export type { ExtractedField }; + +export type DocumentExtractionStatus = "requested" | "completed" | "failed"; + +export interface DocumentExtractionRecord { + extraction_id: string; + document_id: string; + tenant_id: string; + project_id: string; + model_id: string; + status: DocumentExtractionStatus; + /** Every key DI reported, evidence-based: never fabricated, always carries a confidence. */ + fields: Record; + line_items: Array>; + fields_found_count: number; + fields_total_count: number; + average_confidence: number | null; + document_confidence: number | null; + st0re_artifact_id?: string; + n0d_job_id?: string; + error?: string; + correlation_id?: string; + created_at: string; + updated_at?: string; +} diff --git a/src/types/document.ts b/src/types/document.ts new file mode 100644 index 0000000..a0fc186 --- /dev/null +++ b/src/types/document.ts @@ -0,0 +1,27 @@ +export type DocumentStatus = + | "captured" + | "extraction_requested" + | "extracted" + | "extraction_failed"; + +export interface DocumentRecord { + document_id: string; + tenant_id: string; + project_id: string; + mailbox_address: string; + graph_message_id: string; + graph_attachment_id: string; + filename?: string; + content_type?: string; + size_bytes?: number; + received_at?: string; + sender_email?: string; + sender_name?: string; + subject?: string; + st0re_artifact_id?: string; + n0d_job_id?: string; + status: DocumentStatus; + correlation_id?: string; + created_at: string; + updated_at?: string; +} diff --git a/src/types/external-writeback.ts b/src/types/external-writeback.ts new file mode 100644 index 0000000..3e33e85 --- /dev/null +++ b/src/types/external-writeback.ts @@ -0,0 +1,62 @@ +/** + * src/types/external-writeback.ts + * ================================= + * Types for `0pt.write_external_record` -- "POST a validated, already- + * approved structured record to a registered external-system adapter." + * See ../services/external-writeback-service.ts for the service that + * implements this capability and ../adapters/writeback-adapter.ts for the + * adapter interface any external system plugs into. + */ + +export type ExternalWriteStatus = "claimed" | "written" | "failed" | "skipped_duplicate"; + +/** + * One request to write a record to an external system. Deliberately generic + * -- nothing here is Odoo/invoice/Protosell-shaped. `payload` is whatever + * structured record the caller (e.g. an ACT0 workflow's 'write_record' step, + * or a direct API call) has already validated/approved; the registered + * adapter for `adapter_id` is solely responsible for interpreting it. + */ +export interface ExternalWriteRequest { + /** Which registered WritebackAdapter (../adapters/writeback-adapter.ts) handles this write, + * e.g. "odoo_bills". Looked up via ExternalWritebackService.registerAdapter. */ + adapter_id: string; + /** + * Caller-supplied stable key that is the SAME across retries of "the same logical write" -- + * e.g. a document_extraction_id, or an upstream system's own row id. This is what makes + * `writeExternalRecord` idempotent: calling it twice with the same (adapter_id, + * idempotency_key) pair never produces two external writes. See + * ExternalWritebackService's own module doc comment for the exact claim mechanism this + * ports forward from v0-datahub's proven invoice-bridge.ts Postgres-row CLAIM pattern. + */ + idempotency_key: string; + /** Free-form record-type tag the adapter interprets, e.g. "vendor_bill". Not validated by + * the generic service -- purely forwarded to the adapter. */ + record_type: string; + /** The structured record itself -- adapter-specific shape, opaque to the generic service. */ + payload: Readonly>; + tenant_id: string; + project_id: string; + actor_id: string; + correlation_id: string; +} + +/** Persisted record of one write attempt (or claim) -- the generic service's own bookkeeping, + * independent of whatever the adapter itself may also persist. */ +export interface ExternalWriteRecord { + write_id: string; + adapter_id: string; + idempotency_key: string; + record_type: string; + status: ExternalWriteStatus; + /** The external system's own id for the created record (e.g. an Odoo account.move id), + * populated once status is "written". */ + external_id?: string; + error?: string; + tenant_id: string; + project_id: string; + actor_id: string; + correlation_id: string; + created_at: string; + updated_at?: string; +} diff --git a/tests/p0/document-capture-mailbox.test.ts b/tests/p0/document-capture-mailbox.test.ts new file mode 100644 index 0000000..8f1180f --- /dev/null +++ b/tests/p0/document-capture-mailbox.test.ts @@ -0,0 +1,168 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import os from "os"; +import path from "path"; +import fs from "fs"; +import { DocumentCaptureService } from "@/src/services/document-capture-service"; +import { LocalBlobStore } from "@/src/adapters/local-blob-store"; + +function jsonResponse(body: unknown, opts: { status?: number; headers?: Record } = {}) { + return { + ok: (opts.status ?? 200) < 400, + status: opts.status ?? 200, + headers: { get: (k: string) => (opts.headers ?? {})[k] ?? null }, + json: async () => body, + text: async () => JSON.stringify(body), + } as unknown as Response; +} + +describe("DocumentCaptureService.captureFromMailbox", () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "0pt-blob-")); + }); + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + it("downloads PDF attachments, persists bytes, registers a ST0RE artifact, enqueues an N0D extraction job, and emits AUD0 events", async () => { + const pdfBytes = Buffer.from("%PDF-1.4 real bytes for test"); + const pdfBase64 = pdfBytes.toString("base64"); + const aud0Calls: { url: string; body: any }[] = []; + + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (url.includes("login.microsoftonline.com")) return jsonResponse({ access_token: "tok", expires_in: 3600 }); + if (url.includes("/messages?")) { + return jsonResponse({ + value: [ + { + id: "msg-1", + receivedDateTime: "2026-07-01T09:00:00Z", + from: { emailAddress: { name: "Sender", address: "sender@example.com" } }, + toRecipients: [], + subject: "A document", + bodyPreview: "preview", + hasAttachments: true, + webLink: "https://outlook.office.com/mail/1", + }, + ], + }); + } + if (url.includes("/attachments")) { + return jsonResponse({ + value: [ + { + "@odata.type": "#microsoft.graph.fileAttachment", + id: "att-1", + name: "doc.pdf", + contentType: "application/pdf", + size: pdfBytes.length, + contentBytes: pdfBase64, + }, + ], + }); + } + if (url.includes("/v1/aud0/events") || url.includes("/v1/aud0/lineage/events")) { + aud0Calls.push({ url, body: init?.body ? JSON.parse(init.body as string) : null }); + return jsonResponse({}); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + + const service = new DocumentCaptureService({ + blobStore: new LocalBlobStore(tmpDir), + aud0BaseUrl: "https://aud0.example.com", + }); + const result = await service.captureFromMailbox( + { mailbox: { tenantId: "t", clientId: "c", clientSecret: "s", mailboxAddress: "docs@customer.com" } }, + { tenant_id: "tenant1", project_id: "proj1", actor_id: "actor1", correlation_id: "corr1" }, + ); + + expect(result.scanned_messages).toBe(1); + expect(result.captured_documents).toHaveLength(1); + const doc = result.captured_documents[0]; + expect(doc.status).toBe("extraction_requested"); + expect(doc.filename).toBe("doc.pdf"); + expect(doc.mailbox_address).toBe("docs@customer.com"); + expect(doc.st0re_artifact_id).toContain("mock_artifact_"); + expect(doc.n0d_job_id).toContain("mock_job_"); + + // Bytes were actually persisted (INTERIM local store) and are readable back byte-for-byte. + const storedBytes = await service.getDocumentBytes(doc.document_id); + expect(storedBytes.equals(pdfBytes)).toBe(true); + + const captured = aud0Calls.find((c) => c.body?.event_type === "0pt.document_captured"); + expect(captured).toBeTruthy(); + expect(captured!.body.payload.filename).toBe("doc.pdf"); + expect(captured!.body.capability_id).toBe("0pt.capture_document"); + const lineage = aud0Calls.find((c) => c.url.includes("lineage")); + expect(lineage).toBeTruthy(); + expect(lineage!.body.target_id).toBe(doc.document_id); + }); + + it("skips non-PDF attachments and records per-message errors without aborting the whole run", async () => { + const fetchMock = vi.fn(async (url: string) => { + if (url.includes("login.microsoftonline.com")) return jsonResponse({ access_token: "tok", expires_in: 3600 }); + if (url.includes("/messages?")) { + return jsonResponse({ + value: [ + { + id: "msg-ok", + receivedDateTime: "2026-07-01T09:00:00Z", + hasAttachments: true, + from: { emailAddress: {} }, + toRecipients: [], + }, + { + id: "msg-bad", + receivedDateTime: "2026-07-01T09:05:00Z", + hasAttachments: true, + from: { emailAddress: {} }, + toRecipients: [], + }, + ], + }); + } + if (url.includes("msg-ok/attachments")) { + return jsonResponse({ + value: [ + { + "@odata.type": "#microsoft.graph.fileAttachment", + id: "a1", + name: "x.png", + contentType: "image/png", + contentBytes: Buffer.from("x").toString("base64"), + }, + ], + }); + } + if (url.includes("msg-bad/attachments")) { + return { ok: false, status: 500, headers: { get: () => null }, json: async () => ({ error: { code: "Fail", message: "boom" } }) } as unknown as Response; + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + + const service = new DocumentCaptureService({ blobStore: new LocalBlobStore(tmpDir) }); + const result = await service.captureFromMailbox( + { mailbox: { tenantId: "t", clientId: "c", clientSecret: "s", mailboxAddress: "docs@customer.com" } }, + { tenant_id: "tenant1", project_id: "proj1", actor_id: "actor1", correlation_id: "corr1" }, + ); + expect(result.scanned_messages).toBe(2); + expect(result.captured_documents).toHaveLength(0); // png skipped, msg-bad errored + expect(result.errors).toHaveLength(1); + expect(result.errors[0]).toContain("msg-bad"); + }); + + it("rejects when mailbox credentials are incomplete", async () => { + const service = new DocumentCaptureService({ blobStore: new LocalBlobStore(tmpDir) }); + await expect( + service.captureFromMailbox( + { mailbox: { tenantId: "t", clientId: "", clientSecret: "s", mailboxAddress: "docs@customer.com" } }, + { tenant_id: "tenant1", project_id: "proj1", actor_id: "actor1", correlation_id: "corr1" }, + ), + ).rejects.toThrow(/missing mailbox/i); + }); +}); diff --git a/tests/p0/document-extraction-never-fabricates.test.ts b/tests/p0/document-extraction-never-fabricates.test.ts new file mode 100644 index 0000000..aacc526 --- /dev/null +++ b/tests/p0/document-extraction-never-fabricates.test.ts @@ -0,0 +1,171 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import os from "os"; +import path from "path"; +import fs from "fs"; +import { DocumentCaptureService } from "@/src/services/document-capture-service"; +import { DocumentExtractionService } from "@/src/services/document-extraction-service"; +import { LocalBlobStore } from "@/src/adapters/local-blob-store"; + +function jsonResponse(body: unknown, opts: { status?: number; headers?: Record } = {}) { + return { + ok: (opts.status ?? 200) < 400, + status: opts.status ?? 200, + headers: { get: (k: string) => (opts.headers ?? {})[k] ?? null }, + json: async () => body, + text: async () => JSON.stringify(body), + } as unknown as Response; +} + +describe("DocumentExtractionService — evidence-based extraction", () => { + let tmpDir: string; + let aud0Calls: { url: string; body: any }[]; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "0pt-blob-")); + aud0Calls = []; + }); + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + async function seedCapturedDocument(captureService: DocumentCaptureService) { + const pdfBytes = Buffer.from("%PDF-1.4 fake invoice pdf"); + const fetchMock = vi.fn(async (url: string) => { + if (url.includes("login.microsoftonline.com")) return jsonResponse({ access_token: "tok", expires_in: 3600 }); + if (url.includes("/messages?")) { + return jsonResponse({ + value: [{ id: "msg-1", receivedDateTime: "2026-07-01T09:00:00Z", hasAttachments: true, from: { emailAddress: {} }, toRecipients: [] }], + }); + } + if (url.includes("/attachments")) { + return jsonResponse({ + value: [ + { + "@odata.type": "#microsoft.graph.fileAttachment", + id: "att-1", + name: "doc.pdf", + contentType: "application/pdf", + contentBytes: pdfBytes.toString("base64"), + }, + ], + }); + } + throw new Error(`unexpected fetch during seed: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + const result = await captureService.captureFromMailbox( + { mailbox: { tenantId: "t", clientId: "c", clientSecret: "s", mailboxAddress: "docs@customer.com" } }, + { tenant_id: "tenant1", project_id: "proj1", actor_id: "actor1", correlation_id: "corr-seed" }, + ); + return result.captured_documents[0]; + } + + it("never fabricates a value for a field Document Intelligence did not return, and preserves DI's confidence either way", async () => { + const captureService = new DocumentCaptureService({ blobStore: new LocalBlobStore(tmpDir) }); + const document = await seedCapturedDocument(captureService); + + const extractionService = new DocumentExtractionService(captureService, { + blobStore: new LocalBlobStore(tmpDir), + aud0BaseUrl: "https://aud0.example.com", + documentIntelligence: { endpoint: "https://di.example.com", apiKey: "key123", pollIntervalMs: 1 }, + }); + + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (url.includes(":analyze")) { + return jsonResponse({}, { status: 202, headers: { "operation-location": "https://di.example.com/op/abc" } }); + } + if (url.includes("/op/abc")) { + return jsonResponse({ + status: "succeeded", + analyzeResult: { + content: "Acme AB\nTotal: 1234.50 SEK", + pages: [{}], + documents: [ + { + confidence: 0.9, + fields: { + VendorName: { type: "string", valueString: "Acme AB", confidence: 0.97 }, + InvoiceTotal: { type: "currency", valueCurrency: { amount: 1234.5 }, confidence: 0.92 }, + // DueDate was detected as a field region but DI could not read a value — + // this is exactly the case that must NOT be fabricated into a fake date. + DueDate: { type: "date", confidence: 0.31 }, + // CustomerAddress is not present in the DI response at all. + }, + }, + ], + }, + }); + } + if (url.includes("/v1/aud0/events") || url.includes("/v1/aud0/lineage/events")) { + aud0Calls.push({ url, body: init?.body ? JSON.parse(init.body as string) : null }); + return jsonResponse({}); + } + throw new Error(`unexpected fetch during extraction: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + + const extraction = await extractionService.extractDocument(document.document_id, { + tenant_id: "tenant1", + project_id: "proj1", + actor_id: "actor1", + correlation_id: "corr-extract", + model_id: "prebuilt-invoice", + }); + + expect(extraction.status).toBe("completed"); + // Found fields carry the real value + DI's own confidence. + expect(extraction.fields.VendorName).toEqual({ value: "Acme AB", confidence: 0.97, found: true }); + expect(extraction.fields.InvoiceTotal).toEqual({ value: 1234.5, confidence: 0.92, found: true }); + // Not-found field: value is null (never fabricated); confidence is whatever DI reported. + expect(extraction.fields.DueDate).toEqual({ value: null, confidence: 0.31, found: false }); + // A field DI never mentioned at all is simply absent — also never fabricated into existence. + expect(extraction.fields.CustomerAddress).toBeUndefined(); + + expect(extraction.fields_total_count).toBe(3); + expect(extraction.fields_found_count).toBe(2); + // Average confidence is computed only over the found fields — DueDate's 0.31 excluded. + expect(extraction.average_confidence).toBeCloseTo((0.97 + 0.92) / 2, 5); + expect(extraction.document_confidence).toBe(0.9); + expect(extraction.st0re_artifact_id).toContain("mock_artifact_"); + expect(extraction.model_id).toBe("prebuilt-invoice"); + + const updatedDoc = await captureService.getDocument(document.document_id); + expect(updatedDoc.status).toBe("extracted"); + + const extractedEvent = aud0Calls.find((c) => c.body?.event_type === "0pt.document_extracted"); + expect(extractedEvent).toBeTruthy(); + expect(extractedEvent!.body.capability_id).toBe("0pt.extract_document"); + expect(extractedEvent!.body.payload.fields_found_count).toBe(2); + expect(extractedEvent!.body.payload.fields_total_count).toBe(3); + }); + + it("marks the extraction failed — never returns a fabricated success — when Document Intelligence errors", async () => { + const captureService = new DocumentCaptureService({ blobStore: new LocalBlobStore(tmpDir) }); + const document = await seedCapturedDocument(captureService); + + const extractionService = new DocumentExtractionService(captureService, { + blobStore: new LocalBlobStore(tmpDir), + documentIntelligence: { endpoint: "https://di.example.com", apiKey: "key123", pollIntervalMs: 1 }, + }); + + const fetchMock = vi.fn(async (url: string) => { + if (url.includes(":analyze")) return jsonResponse({}, { status: 500 }); + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + + await expect( + extractionService.extractDocument(document.document_id, { + tenant_id: "tenant1", + project_id: "proj1", + actor_id: "actor1", + correlation_id: "corr-fail", + model_id: "prebuilt-invoice", + }), + ).rejects.toThrow(); + + const updatedDoc = await captureService.getDocument(document.document_id); + expect(updatedDoc.status).toBe("extraction_failed"); + }); +}); diff --git a/tests/p0/document-extraction-not-configured.test.ts b/tests/p0/document-extraction-not-configured.test.ts new file mode 100644 index 0000000..662fdeb --- /dev/null +++ b/tests/p0/document-extraction-not-configured.test.ts @@ -0,0 +1,37 @@ +import { describe, it, expect } from "vitest"; +import { DocumentCaptureService } from "@/src/services/document-capture-service"; +import { DocumentExtractionService } from "@/src/services/document-extraction-service"; + +describe("DocumentExtractionService — refuses to fabricate when unconfigured", () => { + it("throws instead of returning an empty/fake success when Document Intelligence has no endpoint/apiKey", async () => { + const captureService = new DocumentCaptureService(); + const extractionService = new DocumentExtractionService(captureService); // no documentIntelligence opt passed + + await expect( + extractionService.extractDocument("doc_does_not_matter", { + tenant_id: "t1", + project_id: "p1", + actor_id: "a1", + correlation_id: "c1", + model_id: "prebuilt-invoice", + }), + ).rejects.toThrow(/not configured/i); + }); + + it("rejects a missing model_id rather than silently picking one", async () => { + const captureService = new DocumentCaptureService(); + const extractionService = new DocumentExtractionService(captureService, { + documentIntelligence: { endpoint: "https://di.example.com", apiKey: "key123" }, + }); + + await expect( + extractionService.extractDocument("doc_does_not_matter", { + tenant_id: "t1", + project_id: "p1", + actor_id: "a1", + correlation_id: "c1", + model_id: "", + }), + ).rejects.toThrow(/model_id/i); + }); +}); diff --git a/tests/p0/document-intelligence-adapter.test.ts b/tests/p0/document-intelligence-adapter.test.ts new file mode 100644 index 0000000..547466d --- /dev/null +++ b/tests/p0/document-intelligence-adapter.test.ts @@ -0,0 +1,136 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { DocumentIntelligenceAdapter } from "@/src/adapters/document-intelligence-adapter"; + +function jsonResponse(body: unknown, opts: { status?: number; headers?: Record } = {}) { + return { + ok: (opts.status ?? 200) < 400, + status: opts.status ?? 200, + headers: { get: (k: string) => (opts.headers ?? {})[k] ?? null }, + json: async () => body, + text: async () => JSON.stringify(body), + } as unknown as Response; +} + +describe("DocumentIntelligenceAdapter", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("isConfigured requires endpoint + apiKey", () => { + expect(DocumentIntelligenceAdapter.isConfigured({ endpoint: "https://x", apiKey: "k" })).toBe(true); + expect(DocumentIntelligenceAdapter.isConfigured({ endpoint: "https://x" })).toBe(false); + expect(DocumentIntelligenceAdapter.isConfigured({})).toBe(false); + }); + + it("normalizes structured + array fields and never fabricates a missing value", async () => { + const fetchMock = vi.fn(async (url: string) => { + if (url.includes(":analyze")) { + return jsonResponse({}, { status: 202, headers: { "operation-location": "https://di.example.com/op/123" } }); + } + if (url.includes("/op/123")) { + return jsonResponse({ + status: "succeeded", + analyzeResult: { + content: "raw extracted text", + pages: [{}, {}], + documents: [ + { + confidence: 0.93, + fields: { + VendorName: { type: "string", valueString: "Acme AB", confidence: 0.98 }, + InvoiceTotal: { type: "currency", valueCurrency: { amount: 1234.5, currencyCode: "SEK" }, confidence: 0.91 }, + // Field region detected but no value could be read — must stay blank, not fabricated. + DueDate: { type: "date", confidence: 0.4 }, + Items: { + type: "array", + valueArray: [ + { + type: "object", + valueObject: { + Description: { type: "string", valueString: "Widget", confidence: 0.9 }, + Amount: { type: "currency", valueCurrency: { amount: 100 }, confidence: 0.85 }, + Unit: { type: "string", confidence: 0.2 }, // present but empty -> found:false + }, + }, + ], + }, + }, + }, + ], + }, + }); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + + const adapter = new DocumentIntelligenceAdapter({ endpoint: "https://di.example.com", apiKey: "key123", pollIntervalMs: 1 }); + const result = await adapter.analyzeDocument(Buffer.from("%PDF-1.4"), "prebuilt-invoice"); + + expect(result.modelId).toBe("prebuilt-invoice"); + expect(result.content).toBe("raw extracted text"); + expect(result.documentConfidence).toBe(0.93); + expect(result.pageCount).toBe(2); + + expect(result.fields.VendorName).toEqual({ value: "Acme AB", confidence: 0.98, found: true }); + expect(result.fields.InvoiceTotal).toEqual({ value: 1234.5, confidence: 0.91, found: true }); + // Not fabricated: value stays null, but DI's own (low) confidence is still recorded. + expect(result.fields.DueDate).toEqual({ value: null, confidence: 0.4, found: false }); + + expect(result.arrayFields.Items).toHaveLength(1); + expect(result.arrayFields.Items[0].Description).toEqual({ value: "Widget", confidence: 0.9, found: true }); + expect(result.arrayFields.Items[0].Amount).toEqual({ value: 100, confidence: 0.85, found: true }); + expect(result.arrayFields.Items[0].Unit).toEqual({ value: null, confidence: 0.2, found: false }); + }); + + it("retries once on 429 then proceeds", async () => { + let analyzeAttempts = 0; + const fetchMock = vi.fn(async (url: string) => { + if (url.includes(":analyze")) { + analyzeAttempts++; + if (analyzeAttempts === 1) return jsonResponse({}, { status: 429 }); + return jsonResponse({}, { status: 202, headers: { "operation-location": "https://di.example.com/op/retry" } }); + } + if (url.includes("/op/retry")) { + return jsonResponse({ status: "succeeded", analyzeResult: { documents: [{ fields: {} }] } }); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + + const adapter = new DocumentIntelligenceAdapter({ + endpoint: "https://di.example.com", + apiKey: "key123", + pollIntervalMs: 1, + rateLimitRetryMs: 1, + }); + const result = await adapter.analyzeDocument(Buffer.from("x"), "prebuilt-receipt"); + expect(analyzeAttempts).toBe(2); + expect(result.fields).toEqual({}); + }); + + it("throws (never returns a fabricated result) when analyze status is failed", async () => { + const fetchMock = vi.fn(async (url: string) => { + if (url.includes(":analyze")) { + return jsonResponse({}, { status: 202, headers: { "operation-location": "https://di.example.com/op/999" } }); + } + if (url.includes("/op/999")) { + return jsonResponse({ status: "failed", error: { code: "InvalidDocument" } }); + } + throw new Error("unexpected"); + }); + vi.stubGlobal("fetch", fetchMock); + const adapter = new DocumentIntelligenceAdapter({ endpoint: "https://di.example.com", apiKey: "key123", pollIntervalMs: 1 }); + await expect(adapter.analyzeDocument(Buffer.from("x"), "prebuilt-invoice")).rejects.toThrow(/failed/i); + }); + + it("throws when the analyze request itself errors (5xx)", async () => { + const fetchMock = vi.fn(async (url: string) => { + if (url.includes(":analyze")) return jsonResponse({}, { status: 500 }); + throw new Error("unexpected"); + }); + vi.stubGlobal("fetch", fetchMock); + const adapter = new DocumentIntelligenceAdapter({ endpoint: "https://di.example.com", apiKey: "key123" }); + await expect(adapter.analyzeDocument(Buffer.from("x"), "prebuilt-invoice")).rejects.toThrow(/500/); + }); +}); diff --git a/tests/p0/external-writeback-service.test.ts b/tests/p0/external-writeback-service.test.ts new file mode 100644 index 0000000..639aef7 --- /dev/null +++ b/tests/p0/external-writeback-service.test.ts @@ -0,0 +1,245 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { ExternalWritebackService } from "@/src/services/external-writeback-service"; +import type { WritebackAdapter, WritebackAdapterContext, WritebackRequest, WritebackResult } from "@/src/adapters/writeback-adapter"; +import type { ExternalWriteRequest } from "@/src/types/external-writeback"; + +function jsonResponse(body: unknown, opts: { status?: number } = {}) { + return { + ok: (opts.status ?? 200) < 400, + status: opts.status ?? 200, + json: async () => body, + text: async () => JSON.stringify(body), + } as unknown as Response; +} + +/** A configurable in-memory WritebackAdapter double -- records every `write` call so tests can + * assert on call count (critical for the idempotency/concurrency tests below), and can be told + * to fail N times before succeeding, or to always fail/always succeed. */ +class TestWritebackAdapter implements WritebackAdapter { + readonly adapterId = "test_adapter"; + calls: Array<{ request: WritebackRequest; ctx: WritebackAdapterContext }> = []; + configured = true; + private failuresRemaining = 0; + private failureError = "test_adapter_failure"; + externalIdPrefix = "ext"; + + isConfigured(): boolean { + return this.configured; + } + + failNextCalls(n: number, error = "test_adapter_failure"): void { + this.failuresRemaining = n; + this.failureError = error; + } + + async write(request: WritebackRequest, ctx: WritebackAdapterContext): Promise { + this.calls.push({ request, ctx }); + if (this.failuresRemaining > 0) { + this.failuresRemaining -= 1; + return { ok: false, error: this.failureError }; + } + return { ok: true, externalId: `${this.externalIdPrefix}_${this.calls.length}` }; + } +} + +function baseRequest(overrides: Partial = {}): ExternalWriteRequest { + return { + adapter_id: "test_adapter", + idempotency_key: "idem-1", + record_type: "test_record", + payload: { foo: "bar" }, + tenant_id: "tenant1", + project_id: "proj1", + actor_id: "actor1", + correlation_id: "corr-1", + ...overrides, + }; +} + +/** Stubs global fetch to answer CL0UD policy evaluation (always "allow" unless `denyOrApproval` + * is set) and swallow AUD0 event/lineage POSTs, recording them for assertions. */ +function stubCl0udAllowAndAud0(opts: { decision?: "allow" | "deny" | "approval_required" } = {}) { + const aud0Calls: { url: string; body: any }[] = []; + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + if (url.includes("/v1/cl0ud/policies/evaluate")) { + return jsonResponse({ + decision_id: "pdec_test", + decision: opts.decision ?? "allow", + reason: opts.decision === "deny" ? "test_deny" : opts.decision === "approval_required" ? "test_approval_required" : undefined, + }); + } + if (url.includes("/v1/aud0/events") || url.includes("/v1/aud0/lineage/events")) { + aud0Calls.push({ url, body: init?.body ? JSON.parse(init.body as string) : null }); + return jsonResponse({}); + } + throw new Error(`unexpected fetch in test: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + return { aud0Calls, fetchMock }; +} + +describe("ExternalWritebackService — adapter resolution", () => { + afterEach(() => vi.restoreAllMocks()); + + it("throws when no adapter is registered for the requested adapter_id", async () => { + stubCl0udAllowAndAud0(); + const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com" }); + await expect(service.writeExternalRecord(baseRequest())).rejects.toThrow(/no adapter registered/); + }); + + it("throws when the registered adapter reports isConfigured() === false, without calling write()", async () => { + stubCl0udAllowAndAud0(); + const adapter = new TestWritebackAdapter(); + adapter.configured = false; + const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com", adapters: [adapter] }); + await expect(service.writeExternalRecord(baseRequest())).rejects.toThrow(/not configured/); + expect(adapter.calls).toHaveLength(0); + }); +}); + +describe("ExternalWritebackService — CL0UD policy gate", () => { + afterEach(() => vi.restoreAllMocks()); + + it("denies the write and marks the record failed when CL0UD denies", async () => { + stubCl0udAllowAndAud0({ decision: "deny" }); + const adapter = new TestWritebackAdapter(); + const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com", adapters: [adapter] }); + await expect(service.writeExternalRecord(baseRequest())).rejects.toThrow(/Policy denied/); + expect(adapter.calls).toHaveLength(0); + const record = await service.findByIdempotencyKey("test_adapter", "idem-1"); + expect(record?.status).toBe("failed"); + }); + + it("does not silently bypass a CL0UD approval_required decision even though ACT0's own approval step already ran upstream", async () => { + stubCl0udAllowAndAud0({ decision: "approval_required" }); + const adapter = new TestWritebackAdapter(); + const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com", adapters: [adapter] }); + await expect(service.writeExternalRecord(baseRequest())).rejects.toThrow(/Policy requires approval/); + expect(adapter.calls).toHaveLength(0); + }); +}); + +describe("ExternalWritebackService — successful write", () => { + afterEach(() => vi.restoreAllMocks()); + + it("writes via the adapter, returns status 'written' with the adapter's external id, and emits AUD0 event + lineage", async () => { + const { aud0Calls } = stubCl0udAllowAndAud0(); + const adapter = new TestWritebackAdapter(); + const service = new ExternalWritebackService({ + cl0udBaseUrl: "https://cl0ud.example.com", + aud0BaseUrl: "https://aud0.example.com", + adapters: [adapter], + }); + + const record = await service.writeExternalRecord(baseRequest()); + + expect(record.status).toBe("written"); + expect(record.external_id).toBe("ext_1"); + expect(adapter.calls).toHaveLength(1); + expect(adapter.calls[0]?.request.recordType).toBe("test_record"); + expect(adapter.calls[0]?.request.payload).toEqual({ foo: "bar" }); + + const writtenEvent = aud0Calls.find((c) => c.body?.event_type === "0pt.external_record_written"); + expect(writtenEvent).toBeTruthy(); + expect(writtenEvent!.body.capability_id).toBe("0pt.write_external_record"); + expect(writtenEvent!.body.risk_level).toBe("L4"); + expect(writtenEvent!.body.payload.external_id).toBe("ext_1"); + + const lineageEvent = aud0Calls.find((c) => c.url.includes("/lineage/")); + expect(lineageEvent).toBeTruthy(); + expect(lineageEvent!.body.source_id).toBe("idem-1"); + expect(lineageEvent!.body.target_id).toBe("ext_1"); + expect(lineageEvent!.body.lineage_type).toBe("produced"); + }); +}); + +describe("ExternalWritebackService — idempotency (sequential re-calls)", () => { + afterEach(() => vi.restoreAllMocks()); + + it("a second sequential call with the same idempotency_key after a successful write is an idempotent no-op — the adapter is never called again", async () => { + stubCl0udAllowAndAud0(); + const adapter = new TestWritebackAdapter(); + const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com", adapters: [adapter] }); + + const first = await service.writeExternalRecord(baseRequest()); + const second = await service.writeExternalRecord(baseRequest()); + + expect(adapter.calls).toHaveLength(1); + expect(second.write_id).toBe(first.write_id); + expect(second.status).toBe("written"); + expect(second.external_id).toBe(first.external_id); + }); + + it("a failed write CAN be retried with the same idempotency_key — this is a deliberate 'claim survives failure for retry' behavior, not a bug", async () => { + stubCl0udAllowAndAud0(); + const adapter = new TestWritebackAdapter(); + adapter.failNextCalls(1, "transient_error"); + const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com", adapters: [adapter] }); + + await expect(service.writeExternalRecord(baseRequest())).rejects.toThrow(/transient_error/); + const failed = await service.findByIdempotencyKey("test_adapter", "idem-1"); + expect(failed?.status).toBe("failed"); + + const retried = await service.writeExternalRecord(baseRequest()); + expect(retried.status).toBe("written"); + expect(adapter.calls).toHaveLength(2); // one failed attempt, one successful retry + }); + + it("different idempotency_key values against the same adapter never collide", async () => { + stubCl0udAllowAndAud0(); + const adapter = new TestWritebackAdapter(); + const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com", adapters: [adapter] }); + + const a = await service.writeExternalRecord(baseRequest({ idempotency_key: "idem-a" })); + const b = await service.writeExternalRecord(baseRequest({ idempotency_key: "idem-b" })); + + expect(adapter.calls).toHaveLength(2); + expect(a.write_id).not.toBe(b.write_id); + expect(a.external_id).not.toBe(b.external_id); + }); +}); + +describe("ExternalWritebackService — idempotency under concurrency (the regression-critical case)", () => { + afterEach(() => vi.restoreAllMocks()); + + it("two concurrent writeExternalRecord calls with the SAME idempotency_key result in exactly ONE adapter.write call — the claim is synchronous and race-free within one process", async () => { + stubCl0udAllowAndAud0(); + const adapter = new TestWritebackAdapter(); + const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com", adapters: [adapter] }); + + const [resultA, resultB] = await Promise.all([ + service.writeExternalRecord(baseRequest()), + service.writeExternalRecord(baseRequest()), + ]); + + // The critical assertion: no matter how the two promises interleave afterwards, the + // external system is only ever actually written to once for this idempotency_key. This is + // the exact property v0-datahub's invoice-bridge.ts's own atomic Postgres UPDATE claim + // protected in production (two concurrent triggers can't both create a Bill) -- see + // external-writeback-service.ts's own module doc comment for how this port achieves the + // same guarantee via a synchronous, pre-first-await, in-memory claim. + expect(adapter.calls).toHaveLength(1); + + // One call wins the claim and completes the real write; the other observes the claim + // in-flight and returns 'skipped_duplicate' without ever touching the adapter. + expect([resultA.status, resultB.status].sort()).toEqual(["skipped_duplicate", "written"]); + }); + + it("three concurrent calls with the same idempotency_key still only ever produce one adapter.write call", async () => { + stubCl0udAllowAndAud0(); + const adapter = new TestWritebackAdapter(); + const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com", adapters: [adapter] }); + + const results = await Promise.all([ + service.writeExternalRecord(baseRequest()), + service.writeExternalRecord(baseRequest()), + service.writeExternalRecord(baseRequest()), + ]); + + expect(adapter.calls).toHaveLength(1); + const writtenCount = results.filter((r) => r.status === "written").length; + const skippedCount = results.filter((r) => r.status === "skipped_duplicate").length; + expect(writtenCount).toBe(1); + expect(skippedCount).toBe(2); + }); +}); diff --git a/tests/p0/fabric-onelake-adapter.test.ts b/tests/p0/fabric-onelake-adapter.test.ts new file mode 100644 index 0000000..8c23b22 --- /dev/null +++ b/tests/p0/fabric-onelake-adapter.test.ts @@ -0,0 +1,158 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { acquireFabricAccessToken, type FabricOneLakeCredentials } from "@/src/adapters/fabric-onelake-adapter"; + +const CREDS: FabricOneLakeCredentials = { tenantId: "tenant-a", clientId: "client-a", clientSecret: "secret-a" }; + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); +} + +describe("acquireFabricAccessToken — OAuth client-credentials flow", () => { + it("posts client_credentials grant to the caller's tenant AAD endpoint with the default Fabric SQL scope", async () => { + let capturedUrl = ""; + let capturedBody = ""; + const fetchMock = vi.fn(async (url: unknown, init: unknown) => { + capturedUrl = String(url); + capturedBody = String((init as RequestInit).body); + return jsonResponse({ access_token: "tok_123", expires_in: 3600 }); + }); + + const token = await acquireFabricAccessToken(CREDS, undefined, fetchMock as unknown as typeof fetch); + + expect(capturedUrl).toBe("https://login.microsoftonline.com/tenant-a/oauth2/v2.0/token"); + const params = new URLSearchParams(capturedBody); + expect(params.get("client_id")).toBe("client-a"); + expect(params.get("client_secret")).toBe("secret-a"); + expect(params.get("grant_type")).toBe("client_credentials"); + expect(params.get("scope")).toBe("https://database.windows.net/.default"); + expect(token.accessToken).toBe("tok_123"); + expect(token.expiresAtEpochMs).toBeGreaterThan(Date.now()); + }); + + it("uses a caller-supplied scope instead of the default (e.g. OneLake storage)", async () => { + let capturedBody = ""; + const fetchMock = vi.fn(async (_url: unknown, init: unknown) => { + capturedBody = String((init as RequestInit).body); + return jsonResponse({ access_token: "tok_storage", expires_in: 3600 }); + }); + + await acquireFabricAccessToken(CREDS, "https://storage.azure.com/.default", fetchMock as unknown as typeof fetch); + + expect(new URLSearchParams(capturedBody).get("scope")).toBe("https://storage.azure.com/.default"); + }); + + it("never hardcodes a tenant — different credentials hit different tenant token URLs", async () => { + const seenUrls: string[] = []; + const fetchMock = vi.fn(async (url: unknown) => { + seenUrls.push(String(url)); + return jsonResponse({ access_token: "t", expires_in: 3600 }); + }); + + await acquireFabricAccessToken({ tenantId: "tenant-x", clientId: "c", clientSecret: "s" }, undefined, fetchMock as unknown as typeof fetch); + await acquireFabricAccessToken({ tenantId: "tenant-y", clientId: "c", clientSecret: "s" }, undefined, fetchMock as unknown as typeof fetch); + + expect(seenUrls).toEqual([ + "https://login.microsoftonline.com/tenant-x/oauth2/v2.0/token", + "https://login.microsoftonline.com/tenant-y/oauth2/v2.0/token", + ]); + }); + + it("throws a descriptive error when AAD rejects the credentials", async () => { + const fetchMock = vi.fn(async () => jsonResponse({ error: "invalid_client", error_description: "bad secret" }, 401)); + await expect(acquireFabricAccessToken(CREDS, undefined, fetchMock as unknown as typeof fetch)).rejects.toThrow(/invalid_client/); + }); + + it("rejects locally (no network call) when tenantId/clientId/clientSecret are missing", async () => { + const fetchMock = vi.fn(); + await expect( + acquireFabricAccessToken({ tenantId: "", clientId: "c", clientSecret: "s" }, undefined, fetchMock as unknown as typeof fetch), + ).rejects.toThrow(/required/); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe("FabricOneLakeAdapter.query — TDS execution via a mocked mssql driver", () => { + const CONFIG = { + credentials: CREDS, + sqlEndpoint: "https://my-endpoint.datawarehouse.fabric.microsoft.com/", + database: "lh_test", + }; + + beforeEach(() => { + vi.resetModules(); + vi.doUnmock("mssql"); + }); + + it("acquires a token, opens a pooled TDS connection with azure-active-directory-access-token auth, runs the query, and returns rows", async () => { + const fetchMock = vi.fn(async () => jsonResponse({ access_token: "tok_query", expires_in: 3600 })); + + const queryMock = vi.fn(async () => ({ recordset: [{ id: 1, name: "Alice" }, { id: 2, name: "Bob" }] })); + const inputMock = vi.fn(); + const connectMock = vi.fn(async () => {}); + const closeMock = vi.fn(async () => {}); + let capturedConfig: Record | null = null; + + class FakeConnectionPool { + constructor(config: Record) { + capturedConfig = config; + } + connect = connectMock; + request() { + return { input: inputMock, query: queryMock }; + } + close = closeMock; + } + + vi.doMock("mssql", () => ({ default: { ConnectionPool: FakeConnectionPool } })); + + const { FabricOneLakeAdapter } = await import("@/src/adapters/fabric-onelake-adapter"); + const adapter = new FabricOneLakeAdapter(CONFIG, fetchMock as unknown as typeof fetch); + const result = await adapter.query("SELECT * FROM dbo.customers WHERE id > @minId", { minId: 1 }); + + expect(result.rows).toEqual([{ id: 1, name: "Alice" }, { id: 2, name: "Bob" }]); + expect(result.rowCount).toBe(2); + expect(result.durationMs).toBeGreaterThanOrEqual(0); + expect(inputMock).toHaveBeenCalledWith("minId", 1); + expect(connectMock).toHaveBeenCalledTimes(1); + expect(capturedConfig).toMatchObject({ + server: "my-endpoint.datawarehouse.fabric.microsoft.com", + database: "lh_test", + authentication: { type: "azure-active-directory-access-token", options: { token: "tok_query" } }, + }); + + await adapter.close(); + expect(closeMock).toHaveBeenCalledTimes(1); + }); + + it("reuses the cached token across calls and only re-authenticates once", async () => { + const fetchMock = vi.fn(async () => jsonResponse({ access_token: "tok_cached", expires_in: 3600 })); + const queryMock = vi.fn(async () => ({ recordset: [] })); + class FakeConnectionPool { + connect = vi.fn(async () => {}); + request() { + return { input: vi.fn(), query: queryMock }; + } + close = vi.fn(async () => {}); + } + vi.doMock("mssql", () => ({ default: { ConnectionPool: FakeConnectionPool } })); + + const { FabricOneLakeAdapter } = await import("@/src/adapters/fabric-onelake-adapter"); + const adapter = new FabricOneLakeAdapter(CONFIG, fetchMock as unknown as typeof fetch); + await adapter.query("SELECT 1"); + await adapter.query("SELECT 2"); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(queryMock).toHaveBeenCalledTimes(2); + }); + + it("throws a clear, actionable error when the mssql driver is not installed/resolvable", async () => { + const fetchMock = vi.fn(async () => jsonResponse({ access_token: "tok_x", expires_in: 3600 })); + // No vi.doMock("mssql", ...) here — this repo does not declare "mssql" as a + // dependency (see fabric-onelake-adapter.ts header), so the real dynamic + // import genuinely fails, exercising the adapter's real error path. + const { FabricOneLakeAdapter } = await import("@/src/adapters/fabric-onelake-adapter"); + const adapter = new FabricOneLakeAdapter(CONFIG, fetchMock as unknown as typeof fetch); + + await expect(adapter.query("SELECT 1")).rejects.toThrow(/mssql.*not installed/i); + }); +}); diff --git a/tests/p0/graph-mailbox-adapter.test.ts b/tests/p0/graph-mailbox-adapter.test.ts new file mode 100644 index 0000000..5ae2f8f --- /dev/null +++ b/tests/p0/graph-mailbox-adapter.test.ts @@ -0,0 +1,149 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { GraphMailboxAdapter } from "@/src/adapters/graph-mailbox-adapter"; + +function jsonResponse(body: unknown, opts: { status?: number; headers?: Record } = {}) { + return { + ok: (opts.status ?? 200) < 400, + status: opts.status ?? 200, + headers: { get: (k: string) => (opts.headers ?? {})[k] ?? null }, + json: async () => body, + text: async () => JSON.stringify(body), + } as unknown as Response; +} + +describe("GraphMailboxAdapter", () => { + const config = { + tenantId: "tenant-1", + clientId: "client-1", + clientSecret: "secret-1", + mailboxAddress: "docs@customer.com", + folder: "inbox", + }; + + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("isConfigured requires tenantId/clientId/clientSecret/mailboxAddress", () => { + expect(GraphMailboxAdapter.isConfigured(config)).toBe(true); + expect(GraphMailboxAdapter.isConfigured({ ...config, clientSecret: "" })).toBe(false); + expect(GraphMailboxAdapter.isConfigured({ ...config, mailboxAddress: "" })).toBe(false); + }); + + it("polls messages across pages, following @odata.nextLink up to maxPages", async () => { + const calls: string[] = []; + const fetchMock = vi.fn(async (url: string) => { + calls.push(url); + if (url.includes("login.microsoftonline.com")) { + return jsonResponse({ access_token: "tok123", expires_in: 3600 }); + } + if (url.includes("/messages?")) { + return jsonResponse({ + value: [ + { + id: "msg-1", + internetMessageId: "", + conversationId: "conv-1", + receivedDateTime: "2026-07-01T10:00:00Z", + from: { emailAddress: { name: "Vendor AB", address: "vendor@example.com" } }, + toRecipients: [{ emailAddress: { address: "docs@customer.com" } }], + subject: "Document 1", + bodyPreview: "preview", + body: { content: "

Hello World

" }, + hasAttachments: true, + webLink: "https://outlook.office.com/mail/1", + }, + ], + "@odata.nextLink": "https://graph.microsoft.com/v1.0/next-page", + }); + } + if (url.includes("next-page")) { + return jsonResponse({ + value: [ + { + id: "msg-2", + receivedDateTime: "2026-07-01T11:00:00Z", + from: { emailAddress: { name: "Vendor 2", address: "vendor2@example.com" } }, + toRecipients: [], + subject: "Document 2", + hasAttachments: false, + }, + ], + }); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + + const adapter = new GraphMailboxAdapter(config); + const messages = await adapter.pollMessages({ top: 50, maxPages: 2 }); + + expect(messages).toHaveLength(2); + expect(messages[0].id).toBe("msg-1"); + expect(messages[0].senderEmail).toBe("vendor@example.com"); + expect(messages[0].bodyText).toBe("Hello World"); + expect(messages[1].id).toBe("msg-2"); + expect(messages[1].hasAttachments).toBe(false); + expect(calls.some((u) => u.includes("mailFolders/inbox/messages"))).toBe(true); + }); + + it("lists only file attachments and flags PDFs by content-type or filename", async () => { + const pdfBase64 = Buffer.from("%PDF-1.4 fake pdf bytes").toString("base64"); + const fetchMock = vi.fn(async (url: string) => { + if (url.includes("login.microsoftonline.com")) { + return jsonResponse({ access_token: "tok123", expires_in: 3600 }); + } + if (url.includes("/attachments")) { + return jsonResponse({ + value: [ + { + "@odata.type": "#microsoft.graph.fileAttachment", + id: "att-1", + name: "invoice.pdf", + contentType: "application/pdf", + size: 123, + contentBytes: pdfBase64, + }, + { + "@odata.type": "#microsoft.graph.fileAttachment", + id: "att-2", + name: "logo.png", + contentType: "image/png", + size: 50, + contentBytes: Buffer.from("not-a-pdf").toString("base64"), + }, + { + // No mime type but a .pdf filename should still be detected. + "@odata.type": "#microsoft.graph.fileAttachment", + id: "att-3", + name: "scan.PDF", + contentType: "application/octet-stream", + size: 999, + contentBytes: pdfBase64, + }, + { + // Non-file attachment (e.g. embedded email) must be skipped entirely. + "@odata.type": "#microsoft.graph.itemAttachment", + id: "att-4", + name: "embedded-email", + }, + ], + }); + } + throw new Error(`unexpected fetch: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + + const adapter = new GraphMailboxAdapter(config); + const attachments = await adapter.listAttachments("msg-1"); + + expect(attachments).toHaveLength(3); // itemAttachment skipped + const pdf = attachments.find((a) => a.name === "invoice.pdf")!; + expect(pdf.isPdf).toBe(true); + expect(pdf.contentBytes.toString()).toContain("%PDF-1.4"); + const png = attachments.find((a) => a.name === "logo.png")!; + expect(png.isPdf).toBe(false); + const scanByExt = attachments.find((a) => a.name === "scan.PDF")!; + expect(scanByExt.isPdf).toBe(true); + }); +}); diff --git a/tests/p0/odoo-bills-adapter.test.ts b/tests/p0/odoo-bills-adapter.test.ts new file mode 100644 index 0000000..db784fe --- /dev/null +++ b/tests/p0/odoo-bills-adapter.test.ts @@ -0,0 +1,298 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { OdooBillsAdapter, matchPartnerViaFabric, type OdooBillPayload } from "@/src/adapters/odoo-bills-adapter"; +import { FabricOneLakeAdapter } from "@/src/adapters/fabric-onelake-adapter"; +import type { WritebackAdapterContext } from "@/src/adapters/writeback-adapter"; + +function jsonResponse(body: unknown, opts: { status?: number } = {}) { + return { + ok: (opts.status ?? 200) < 400, + status: opts.status ?? 200, + json: async () => body, + text: async () => JSON.stringify(body), + } as unknown as Response; +} + +const CTX: WritebackAdapterContext = { + tenant_id: "t_protosell", + project_id: "proj_ap", + actor_id: "actor1", + correlation_id: "corr-1", +}; + +function basePayload(overrides: Partial = {}): OdooBillPayload { + return { + vendor_name: "Acme AB", + vendor_org_nr: "556123-4567", + invoice_number: "INV-2026-001", + amount_gross: 1234.5, + amount_excl_vat: 987.6, + vat: 246.9, + currency: "SEK", + invoice_date: "2026-07-01", + ...overrides, + }; +} + +describe("OdooBillsAdapter — isConfigured", () => { + it("is not configured without a baseUrl", () => { + const adapter = new OdooBillsAdapter({ baseUrl: "", apiKey: "key" }); + expect(adapter.isConfigured()).toBe(false); + }); + + it("is not configured with a baseUrl but no apiKey and no allowUnauthenticated flag", () => { + const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com" }); + expect(adapter.isConfigured()).toBe(false); + }); + + it("is configured with baseUrl + apiKey", () => { + const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key" }); + expect(adapter.isConfigured()).toBe(true); + }); + + it("is configured with baseUrl + explicit allowUnauthenticated, without an apiKey", () => { + const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", allowUnauthenticated: true }); + expect(adapter.isConfigured()).toBe(true); + }); + + it("write() returns ok:false when not configured, without attempting a network call", async () => { + const fetchMock = vi.fn(); + const adapter = new OdooBillsAdapter({ baseUrl: "", fetchImpl: fetchMock as unknown as typeof fetch }); + const result = await adapter.write({ recordType: "vendor_bill", payload: basePayload() as unknown as Record, idempotencyKey: "k1" }, CTX); + expect(result.ok).toBe(false); + expect(result.error).toBe("odoo_not_configured"); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe("OdooBillsAdapter — guards ported verbatim from invoice-bridge.ts", () => { + afterEach(() => vi.restoreAllMocks()); + + it("refuses to post a payment reminder (is_reminder: true) — mirrors 'Påminnelse postas inte till Odoo'", async () => { + const fetchMock = vi.fn(); + const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch }); + const result = await adapter.write( + { recordType: "vendor_bill", payload: basePayload({ is_reminder: true }) as unknown as Record, idempotencyKey: "k1" }, + CTX, + ); + expect(result).toEqual({ ok: false, error: "is_reminder" }); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("refuses to post a detected duplicate (duplicate_flag: true) — mirrors 'Dubblett postas inte till Odoo'", async () => { + const fetchMock = vi.fn(); + const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch }); + const result = await adapter.write( + { recordType: "vendor_bill", payload: basePayload({ duplicate_flag: true }) as unknown as Record, idempotencyKey: "k1" }, + CTX, + ); + expect(result).toEqual({ ok: false, error: "duplicate" }); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("treats a payload with existing_external_id as already posted — idempotent no-op, no network call, mirrors 'Redan bokförd i Odoo... hoppar över'", async () => { + const fetchMock = vi.fn(); + const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch }); + const result = await adapter.write( + { recordType: "vendor_bill", payload: basePayload({ existing_external_id: "999" }) as unknown as Record, idempotencyKey: "k1" }, + CTX, + ); + expect(result).toEqual({ ok: true, externalId: "999" }); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("rejects a payload missing required fields without attempting a network call", async () => { + const fetchMock = vi.fn(); + const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch }); + const result = await adapter.write( + { recordType: "vendor_bill", payload: { vendor_name: "Acme AB" }, idempotencyKey: "k1" }, + CTX, + ); + expect(result.ok).toBe(false); + expect(result.error).toBe("invalid_payload_missing_required_fields"); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe("OdooBillsAdapter — real POST shape (ported from invoice-bridge.ts's pushInvoiceToOdoo)", () => { + afterEach(() => vi.restoreAllMocks()); + + it("POSTs account.move with itemized lines when payload.lines is present", async () => { + let capturedUrl = ""; + let capturedInit: RequestInit | undefined; + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + capturedUrl = url; + capturedInit = init; + return jsonResponse({ ok: true, odoo_id: 4242 }); + }); + const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "sekret", fetchImpl: fetchMock as unknown as typeof fetch }); + + const payload = basePayload({ + partner_id: 77, + lines: [{ description: "Konsulttjänst", quantity: 2, unit_price: 500 }], + }); + const result = await adapter.write({ recordType: "vendor_bill", payload: payload as unknown as Record, idempotencyKey: "k1" }, CTX); + + expect(result).toEqual({ ok: true, externalId: "4242" }); + expect(capturedUrl).toBe("https://odoo.example.com/api/odoo/invoices/write"); + expect((capturedInit?.headers as Record).Authorization).toBe("Bearer sekret"); + const body = JSON.parse(capturedInit!.body as string); + expect(body.model).toBe("account.move"); + expect(body.invoice.partner_id).toBe(77); + expect(body.invoice.ref).toBe("INV-2026-001"); + expect(body.invoice.invoice_line_ids).toEqual([{ name: "Konsulttjänst", quantity: 2, price_unit: 500 }]); + }); + + it("synthesizes a single summary line from description/vendor_name when payload.lines is absent — ported verbatim from processInboxToOdoo's line-synthesis fallback", async () => { + let capturedInit: RequestInit | undefined; + const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => { + capturedInit = init; + return jsonResponse({ ok: true, odoo_id: 100 }); + }); + const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch }); + + const payload = basePayload({ description: "Städtjänster juni", amount_excl_vat: 800 }); + await adapter.write({ recordType: "vendor_bill", payload: payload as unknown as Record, idempotencyKey: "k1" }, CTX); + + const body = JSON.parse(capturedInit!.body as string); + expect(body.invoice.invoice_line_ids).toEqual([{ name: "Städtjänster juni", quantity: 1, price_unit: 800 }]); + }); + + it("falls back to vendor_name, then a synthesized 'Faktura ' string, when description is absent", async () => { + let capturedInit: RequestInit | undefined; + const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => { + capturedInit = init; + return jsonResponse({ ok: true, odoo_id: 101 }); + }); + const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch }); + + const payload = basePayload({ vendor_name: "", invoice_number: "INV-777", amount_excl_vat: 50 }); + await adapter.write({ recordType: "vendor_bill", payload: payload as unknown as Record, idempotencyKey: "k1" }, CTX); + + const body = JSON.parse(capturedInit!.body as string); + expect(body.invoice.invoice_line_ids[0].name).toBe("Faktura INV-777"); + }); + + it("returns ok:false with an odoo_http_ error on a non-2xx response", async () => { + const fetchMock = vi.fn(async () => jsonResponse({}, { status: 500 })); + const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch }); + const result = await adapter.write({ recordType: "vendor_bill", payload: basePayload() as unknown as Record, idempotencyKey: "k1" }, CTX); + expect(result).toEqual({ ok: false, error: "odoo_http_500" }); + }); + + it("returns ok:false when Odoo responds 200 but ok:false/no odoo_id", async () => { + const fetchMock = vi.fn(async () => jsonResponse({ ok: false, error: "odoo_validation_error" })); + const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch }); + const result = await adapter.write({ recordType: "vendor_bill", payload: basePayload() as unknown as Record, idempotencyKey: "k1" }, CTX); + expect(result).toEqual({ ok: false, error: "odoo_validation_error" }); + }); + + it("returns ok:false on a network error, without throwing", async () => { + const fetchMock = vi.fn(async () => { + throw new Error("ECONNREFUSED"); + }); + const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch }); + const result = await adapter.write({ recordType: "vendor_bill", payload: basePayload() as unknown as Record, idempotencyKey: "k1" }, CTX); + expect(result.ok).toBe(false); + expect(result.error).toContain("odoo_network_error"); + }); +}); + +describe("matchPartnerViaFabric — ported from invoice-bridge.ts's matchInvoiceToOdooPartner", () => { + it("matches by digits-only comparison so '556123-4567' resolves the same as a plain digit string", async () => { + const queryMock = vi.fn(async () => ({ + rows: [{ id: 42, name: "Acme AB", vat: "SE5561234567 01" }], + rowCount: 1, + durationMs: 1, + })); + const fabric = { query: queryMock } as unknown as FabricOneLakeAdapter; + + const result = await matchPartnerViaFabric(fabric, "556123-4567"); + expect(result).toEqual({ partner_id: 42, partner_name: "Acme AB" }); + expect(queryMock).toHaveBeenCalledWith(expect.stringContaining("odoo_export__res_partner"), { digits: "5561234567" }); + }); + + it("returns {} (no match) when the org-nr has no digits at all", async () => { + const queryMock = vi.fn(); + const fabric = { query: queryMock } as unknown as FabricOneLakeAdapter; + const result = await matchPartnerViaFabric(fabric, "N/A"); + expect(result).toEqual({}); + expect(queryMock).not.toHaveBeenCalled(); + }); + + it("degrades to {} (no match) rather than throwing when the warehouse query fails — mirrors invoice-bridge.ts's own catch-all", async () => { + const queryMock = vi.fn(async () => { + throw new Error("warehouse unreachable"); + }); + const fabric = { query: queryMock } as unknown as FabricOneLakeAdapter; + const result = await matchPartnerViaFabric(fabric, "556123-4567"); + expect(result).toEqual({}); + }); + + it("returns {} when no partner row matches", async () => { + const queryMock = vi.fn(async () => ({ rows: [], rowCount: 0, durationMs: 1 })); + const fabric = { query: queryMock } as unknown as FabricOneLakeAdapter; + const result = await matchPartnerViaFabric(fabric, "556123-4567"); + expect(result).toEqual({}); + }); +}); + +describe("OdooBillsAdapter — partner resolution wired into write()", () => { + afterEach(() => vi.restoreAllMocks()); + + it("uses payload.partner_id directly when already set, without calling matchPartner", async () => { + const matchPartner = vi.fn(); + const fetchMock = vi.fn(async () => jsonResponse({ ok: true, odoo_id: 1 })); + const adapter = new OdooBillsAdapter({ + baseUrl: "https://odoo.example.com", + apiKey: "key", + matchPartner, + fetchImpl: fetchMock as unknown as typeof fetch, + }); + await adapter.write( + { recordType: "vendor_bill", payload: basePayload({ partner_id: 55 }) as unknown as Record, idempotencyKey: "k1" }, + CTX, + ); + expect(matchPartner).not.toHaveBeenCalled(); + }); + + it("calls the injected matchPartner when partner_id is absent but vendor_org_nr is present, and forwards the resolved partner_id to Odoo", async () => { + const matchPartner = vi.fn(async () => ({ partner_id: 99, partner_name: "Acme AB" })); + let capturedBody: any; + const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => { + capturedBody = JSON.parse(init!.body as string); + return jsonResponse({ ok: true, odoo_id: 1 }); + }); + const adapter = new OdooBillsAdapter({ + baseUrl: "https://odoo.example.com", + apiKey: "key", + matchPartner, + fetchImpl: fetchMock as unknown as typeof fetch, + }); + await adapter.write( + { recordType: "vendor_bill", payload: basePayload() as unknown as Record, idempotencyKey: "k1" }, + CTX, + ); + expect(matchPartner).toHaveBeenCalledWith("556123-4567", CTX); + expect(capturedBody.invoice.partner_id).toBe(99); + }); + + it("posts without a partner_id when no match is found — 'skickar utan partner-koppling'", async () => { + const matchPartner = vi.fn(async () => ({})); + let capturedBody: any; + const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => { + capturedBody = JSON.parse(init!.body as string); + return jsonResponse({ ok: true, odoo_id: 1 }); + }); + const adapter = new OdooBillsAdapter({ + baseUrl: "https://odoo.example.com", + apiKey: "key", + matchPartner, + fetchImpl: fetchMock as unknown as typeof fetch, + }); + await adapter.write( + { recordType: "vendor_bill", payload: basePayload() as unknown as Record, idempotencyKey: "k1" }, + CTX, + ); + expect(capturedBody.invoice.partner_id).toBeUndefined(); + }); +}); diff --git a/tests/p0/source-query-service.test.ts b/tests/p0/source-query-service.test.ts new file mode 100644 index 0000000..9410c07 --- /dev/null +++ b/tests/p0/source-query-service.test.ts @@ -0,0 +1,314 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { SourceService } from "@/src/services/source-service"; +import type { SourceType } from "@/src/types/source"; + +const queryMock = vi.fn(); +const closeMock = vi.fn().mockResolvedValue(undefined); + +// Hoisted by Vitest above the static imports below, so +// source-query-service.ts (which statically imports FabricOneLakeAdapter) +// sees this mock too — no real mssql/AAD network calls happen in this file. +vi.mock("@/src/adapters/fabric-onelake-adapter", () => { + return { + // Must be a real `function` (not an arrow fn) so `new FabricOneLakeAdapter(...)` + // in source-query-service.ts works: a constructor that explicitly returns an + // object substitutes that object for `this`, per JS semantics. + FabricOneLakeAdapter: vi.fn().mockImplementation(function FakeFabricOneLakeAdapter() { + return { query: queryMock, close: closeMock }; + }), + }; +}); + +import { + SourceQueryService, + SourceQueryTimeoutError, + SourceQueryUnsupportedError, + SOURCE_QUERY_HARD_MAX_ROWS, + SOURCE_QUERY_HARD_MAX_TIMEOUT_MS, +} from "@/src/services/source-query-service"; + +const BASE_CONNECTION_CONFIG = { + credentials: { tenantId: "t", clientId: "c", clientSecret: "s" }, + sqlEndpoint: "ep.datawarehouse.fabric.microsoft.com", + database: "lh_test", +}; + +async function registerSource( + sourceService: SourceService, + opts: { source_id: string; adapter_id?: string; source_type?: SourceType }, +): Promise { + await sourceService.createSource({ + source_id: opts.source_id, + tenant_id: "t1", + project_id: "p1", + source_type: opts.source_type ?? "database", + name: "Fabric Warehouse", + adapter_id: opts.adapter_id ?? "fabric_onelake_sql", + }); + return opts.source_id; +} + +describe("SourceQueryService.querySource — bounded execution", () => { + beforeEach(() => { + queryMock.mockReset(); + closeMock.mockClear(); + }); + + it("executes a query against a registered fabric_onelake_sql database source and returns rows", async () => { + const sourceService = new SourceService(); + const sourceId = await registerSource(sourceService, { source_id: "src_query_ok" }); + queryMock.mockResolvedValue({ rows: [{ id: 1 }, { id: 2 }], rowCount: 2, durationMs: 5 }); + + const service = new SourceQueryService({ sourceService }); + const result = await service.querySource({ + source_id: sourceId, + query: "SELECT * FROM dbo.customers", + tenant_id: "t1", + project_id: "p1", + connection_config: BASE_CONNECTION_CONFIG, + }); + + expect(result.rows).toEqual([{ id: 1 }, { id: 2 }]); + expect(result.row_count).toBe(2); + expect(result.truncated).toBe(false); + expect(result.bounds.max_rows).toBeGreaterThan(0); + expect(closeMock).toHaveBeenCalledTimes(1); + }); + + it("rejects a non-database source with SourceQueryUnsupportedError", async () => { + const sourceService = new SourceService(); + const sourceId = await registerSource(sourceService, { source_id: "src_file_1", source_type: "file", adapter_id: undefined }); + + const service = new SourceQueryService({ sourceService }); + await expect( + service.querySource({ source_id: sourceId, query: "SELECT 1", tenant_id: "t1", project_id: "p1", connection_config: BASE_CONNECTION_CONFIG }), + ).rejects.toBeInstanceOf(SourceQueryUnsupportedError); + }); + + it("rejects a database source with an unsupported adapter_id", async () => { + const sourceService = new SourceService(); + const sourceId = await registerSource(sourceService, { source_id: "src_bad_adapter", adapter_id: "some_other_adapter" }); + + const service = new SourceQueryService({ sourceService }); + await expect( + service.querySource({ source_id: sourceId, query: "SELECT 1", tenant_id: "t1", project_id: "p1", connection_config: BASE_CONNECTION_CONFIG }), + ).rejects.toBeInstanceOf(SourceQueryUnsupportedError); + }); + + it("propagates a 'not found' error for an unregistered source, without ever building an adapter", async () => { + const sourceService = new SourceService(); + const service = new SourceQueryService({ sourceService }); + await expect( + service.querySource({ source_id: "does_not_exist", query: "SELECT 1", tenant_id: "t1", project_id: "p1", connection_config: BASE_CONNECTION_CONFIG }), + ).rejects.toThrow(/not found/i); + expect(queryMock).not.toHaveBeenCalled(); + }); + + it("truncates rows to max_rows and flags truncated:true", async () => { + const sourceService = new SourceService(); + const sourceId = await registerSource(sourceService, { source_id: "src_truncate" }); + const rows = Array.from({ length: 50 }, (_, i) => ({ id: i })); + queryMock.mockResolvedValue({ rows, rowCount: rows.length, durationMs: 3 }); + + const service = new SourceQueryService({ sourceService }); + const result = await service.querySource({ + source_id: sourceId, + query: "SELECT * FROM dbo.big_table", + tenant_id: "t1", + project_id: "p1", + max_rows: 10, + connection_config: BASE_CONNECTION_CONFIG, + }); + + expect(result.rows).toHaveLength(10); + expect(result.row_count).toBe(10); + expect(result.truncated).toBe(true); + expect(result.bounds.max_rows).toBe(10); + }); + + it("does not flag truncated when row count is exactly max_rows", async () => { + const sourceService = new SourceService(); + const sourceId = await registerSource(sourceService, { source_id: "src_exact" }); + const rows = Array.from({ length: 10 }, (_, i) => ({ id: i })); + queryMock.mockResolvedValue({ rows, rowCount: rows.length, durationMs: 1 }); + + const service = new SourceQueryService({ sourceService }); + const result = await service.querySource({ + source_id: sourceId, + query: "SELECT * FROM dbo.t", + tenant_id: "t1", + project_id: "p1", + max_rows: 10, + connection_config: BASE_CONNECTION_CONFIG, + }); + + expect(result.truncated).toBe(false); + expect(result.rows).toHaveLength(10); + }); + + it("clamps a caller-requested max_rows above the hard ceiling instead of allowing an unbounded read", async () => { + const sourceService = new SourceService(); + const sourceId = await registerSource(sourceService, { source_id: "src_clamp_rows" }); + queryMock.mockResolvedValue({ rows: [{ id: 1 }], rowCount: 1, durationMs: 1 }); + + const service = new SourceQueryService({ sourceService }); + const result = await service.querySource({ + source_id: sourceId, + query: "SELECT 1", + tenant_id: "t1", + project_id: "p1", + max_rows: SOURCE_QUERY_HARD_MAX_ROWS * 10, + connection_config: BASE_CONNECTION_CONFIG, + }); + + expect(result.bounds.max_rows).toBe(SOURCE_QUERY_HARD_MAX_ROWS); + }); + + it("clamps a caller-requested timeout_ms above the hard ceiling", async () => { + const sourceService = new SourceService(); + const sourceId = await registerSource(sourceService, { source_id: "src_clamp_timeout" }); + queryMock.mockResolvedValue({ rows: [], rowCount: 0, durationMs: 1 }); + + const service = new SourceQueryService({ sourceService }); + const result = await service.querySource({ + source_id: sourceId, + query: "SELECT 1", + tenant_id: "t1", + project_id: "p1", + timeout_ms: SOURCE_QUERY_HARD_MAX_TIMEOUT_MS * 100, + connection_config: BASE_CONNECTION_CONFIG, + }); + + expect(result.bounds.timeout_ms).toBe(SOURCE_QUERY_HARD_MAX_TIMEOUT_MS); + }); + + it("times out a query that runs longer than the bounded timeout, with SourceQueryTimeoutError", async () => { + const sourceService = new SourceService(); + const sourceId = await registerSource(sourceService, { source_id: "src_timeout" }); + queryMock.mockImplementation( + () => new Promise((resolve) => setTimeout(() => resolve({ rows: [], rowCount: 0, durationMs: 1 }), 300)), + ); + + const service = new SourceQueryService({ sourceService }); + await expect( + service.querySource({ + source_id: sourceId, + query: "SELECT * FROM dbo.slow_view", + tenant_id: "t1", + project_id: "p1", + timeout_ms: 20, + connection_config: BASE_CONNECTION_CONFIG, + }), + ).rejects.toBeInstanceOf(SourceQueryTimeoutError); + }); + + it("still releases the adapter (calls close) after a timeout", async () => { + const sourceService = new SourceService(); + const sourceId = await registerSource(sourceService, { source_id: "src_timeout_cleanup" }); + queryMock.mockImplementation( + () => new Promise((resolve) => setTimeout(() => resolve({ rows: [], rowCount: 0, durationMs: 1 }), 300)), + ); + + const service = new SourceQueryService({ sourceService }); + await expect( + service.querySource({ + source_id: sourceId, + query: "SELECT 1", + tenant_id: "t1", + project_id: "p1", + timeout_ms: 20, + connection_config: BASE_CONNECTION_CONFIG, + }), + ).rejects.toThrow(); + + expect(closeMock).toHaveBeenCalledTimes(1); + }); + + it("propagates adapter query errors as-is", async () => { + const sourceService = new SourceService(); + const sourceId = await registerSource(sourceService, { source_id: "src_adapter_error" }); + queryMock.mockRejectedValue(new Error("Invalid object name 'dbo.does_not_exist'")); + + const service = new SourceQueryService({ sourceService }); + await expect( + service.querySource({ + source_id: sourceId, + query: "SELECT * FROM dbo.does_not_exist", + tenant_id: "t1", + project_id: "p1", + connection_config: BASE_CONNECTION_CONFIG, + }), + ).rejects.toThrow(/Invalid object name/); + }); +}); + +describe("SourceQueryService — AUD0 0pt.source_queried event emission", () => { + const originalFetch = global.fetch; + + afterEach(() => { + global.fetch = originalFetch; + }); + + it("emits a 0pt.source_queried AUD0 event with ok:true on success", async () => { + queryMock.mockReset(); + closeMock.mockClear(); + const sourceService = new SourceService(); + const sourceId = await registerSource(sourceService, { source_id: "src_aud0_ok" }); + queryMock.mockResolvedValue({ rows: [{ id: 1 }], rowCount: 1, durationMs: 2 }); + + const calls: Array<{ url: string; body: Record }> = []; + global.fetch = vi.fn(async (url: unknown, init: unknown) => { + calls.push({ url: String(url), body: JSON.parse(String((init as RequestInit).body)) }); + return new Response("{}", { status: 200 }); + }) as unknown as typeof fetch; + + const service = new SourceQueryService({ sourceService, aud0BaseUrl: "http://aud0.test" }); + await service.querySource({ + source_id: sourceId, + query: "SELECT * FROM dbo.customers", + tenant_id: "t1", + project_id: "p1", + actor_id: "user_42", + correlation_id: "corr_abc", + connection_config: BASE_CONNECTION_CONFIG, + }); + + expect(calls).toHaveLength(1); + expect(calls[0].url).toBe("http://aud0.test/v1/aud0/events"); + expect(calls[0].body.event_type).toBe("0pt.source_queried"); + expect(calls[0].body.capability_id).toBe("0pt.query_source"); + expect(calls[0].body.app_id).toBe("0pt"); + expect(calls[0].body.resource_type).toBe("source"); + expect(calls[0].body.resource_id).toBe(sourceId); + expect(calls[0].body.actor_id).toBe("user_42"); + expect(calls[0].body.correlation_id).toBe("corr_abc"); + const payload = calls[0].body.payload as Record; + expect(payload.ok).toBe(true); + expect(payload.row_count).toBe(1); + expect(payload.adapter_id).toBe("fabric_onelake_sql"); + }); + + it("emits a 0pt.source_queried AUD0 event with ok:false and the error message on failure", async () => { + queryMock.mockReset(); + closeMock.mockClear(); + const sourceService = new SourceService(); + const sourceId = await registerSource(sourceService, { source_id: "src_aud0_fail" }); + queryMock.mockRejectedValue(new Error("boom")); + + const calls: Array<{ url: string; body: Record }> = []; + global.fetch = vi.fn(async (url: unknown, init: unknown) => { + calls.push({ url: String(url), body: JSON.parse(String((init as RequestInit).body)) }); + return new Response("{}", { status: 200 }); + }) as unknown as typeof fetch; + + const service = new SourceQueryService({ sourceService, aud0BaseUrl: "http://aud0.test" }); + await expect( + service.querySource({ source_id: sourceId, query: "SELECT 1", tenant_id: "t1", project_id: "p1", connection_config: BASE_CONNECTION_CONFIG }), + ).rejects.toThrow("boom"); + + expect(calls).toHaveLength(1); + const payload = calls[0].body.payload as Record; + expect(payload.ok).toBe(false); + expect(payload.error).toBe("boom"); + }); +});