/** * src/services/document-extraction-service.ts * ============================================== * Evidence-based field extraction for a captured document. * * Calls DocumentIntelligenceAdapter and NEVER fabricates a value: a field * key is only populated with a non-null value when Document Intelligence * actually returned one, and it always carries DI's own confidence score * (or null when DI reported none). Fields DI did not find are left blank — * { value: null, confidence: , found: false } — * never defaulted, guessed, or backfilled from anything else. This mirrors * the evidence-based discipline in * v0-datahub/lib/invoice-inbox/doc-intelligence.ts exactly (it never * invented an amount/vendor/date that DI didn't return); it is a * correctness property, not a style preference, so it is enforced twice — * once inside the adapter's field normalizer, and again defensively here. * * modelId is caller-supplied ("prebuilt-invoice", "prebuilt-receipt", * "prebuilt-layout", or a custom model id) — this service does not assume * any particular document type. Invoice-specific reasoning (vendor * matching, dup-hash, DQ scoring, invoice classification) was intentionally * left in v0-datahub/lib/invoice-inbox and is NOT ported here. */ import crypto from "crypto"; import { DocumentIntelligenceAdapter, type DocumentIntelligenceConfig, type ExtractedField as AdapterField, } from "../adapters/document-intelligence-adapter"; import { LocalBlobStore } from "../adapters/local-blob-store"; import { createCl0udClient, type Cl0udClient } from "../clients/cl0ud-client"; import { createSt0reClient, type St0reClient } from "../clients/st0re-client"; import { createAud0Client, type Aud0Client } from "../clients/aud0-client"; import { DocumentCaptureService, documentCaptureService } from "./document-capture-service"; import type { DocumentExtractionRecord, ExtractedField } from "../types/document-extraction"; export interface ExtractDocumentContext { tenant_id: string; project_id: string; actor_id: string; correlation_id: string; /** e.g. "prebuilt-invoice" | "prebuilt-receipt" | "prebuilt-layout" | a custom model id. */ model_id: string; } /** * Defensive re-application of the "never fabricate" rule at the service * boundary. Even though the adapter's normalizeField() already enforces * this, a field is only ever considered found here if it is BOTH marked * found by the adapter AND carries a genuinely non-empty value — so a * malformed/partial adapter response can never leak a fabricated value * through this service. */ function toEvidenceFields(fields: Record): Record { const out: Record = {}; for (const [key, f] of Object.entries(fields)) { const found = !!f?.found && f.value !== null && f.value !== undefined && f.value !== ""; out[key] = { value: found ? f.value : null, confidence: f?.confidence ?? null, found }; } return out; } export class DocumentExtractionService { private extractions: Map = new Map(); private cl0ud: Cl0udClient; private st0re: St0reClient; private aud0: Aud0Client; private blobStore: LocalBlobStore; private di: DocumentIntelligenceAdapter | null; constructor( private captureService: DocumentCaptureService, opts?: { cl0udBaseUrl?: string; st0reBaseUrl?: string; aud0BaseUrl?: string; documentIntelligence?: DocumentIntelligenceConfig; blobStore?: LocalBlobStore; }, ) { this.cl0ud = createCl0udClient(opts?.cl0udBaseUrl); this.st0re = createSt0reClient(opts?.st0reBaseUrl); this.aud0 = createAud0Client(opts?.aud0BaseUrl); this.blobStore = opts?.blobStore ?? new LocalBlobStore(); this.di = opts?.documentIntelligence && DocumentIntelligenceAdapter.isConfigured(opts.documentIntelligence) ? new DocumentIntelligenceAdapter(opts.documentIntelligence) : null; } async extractDocument(documentId: string, ctx: ExtractDocumentContext): Promise { if (!this.di) { throw new Error( "DocumentExtractionService: Document Intelligence is not configured (missing endpoint/apiKey) — refusing to fabricate a result", ); } if (!ctx.model_id) { throw new Error("extractDocument: missing model_id"); } const document = await this.captureService.getDocument(documentId); const policy = await this.cl0ud.evaluatePolicy({ tenant_id: ctx.tenant_id, project_id: ctx.project_id, actor_id: ctx.actor_id, capability_id: "0pt.extract_document", resource_type: "document", resource_id: documentId, risk_level: "L3", correlation_id: ctx.correlation_id, }); if (policy.decision === "deny") throw new Error(`Policy denied: ${policy.reason}`); if (policy.decision === "approval_required") throw new Error(`Policy requires approval: ${policy.reason}`); const extractionId = `dex_${crypto.randomUUID()}`; const now = new Date().toISOString(); let record: DocumentExtractionRecord = { extraction_id: extractionId, document_id: documentId, tenant_id: ctx.tenant_id, project_id: ctx.project_id, model_id: ctx.model_id, status: "requested", fields: {}, line_items: [], fields_found_count: 0, fields_total_count: 0, average_confidence: null, document_confidence: null, correlation_id: ctx.correlation_id, created_at: now, }; this.extractions.set(extractionId, record); try { const bytes = await this.captureService.getDocumentBytes(documentId); const diResult = await this.di.analyzeDocument(bytes, ctx.model_id); const fields = toEvidenceFields(diResult.fields); const lineItems = Object.values(diResult.arrayFields).flatMap((items) => items.map((item) => toEvidenceFields(item))); const foundCount = Object.values(fields).filter((f) => f.found).length; // Average only over FOUND fields — a not-found field's confidence (e.g. "31% sure this // region isn't a date after all") describes DI's certainty about absence, not a value, // and must not drag down the confidence of the values that were actually extracted. const confidences = Object.values(fields) .filter((f) => f.found) .map((f) => f.confidence) .filter((c): c is number => c != null); const averageConfidence = confidences.length ? confidences.reduce((a, b) => a + b, 0) / confidences.length : null; // Persist the extraction result as an artifact too — same interim-storage gap as // document-capture-service.ts (see that file's header for the full explanation). const payloadJson = Buffer.from( JSON.stringify({ model_id: ctx.model_id, fields, line_items: lineItems, content: diResult.content }), "utf8", ); const bucket = "0pt-document-extractions"; const key = `${ctx.tenant_id}/${ctx.project_id}/${documentId}/${extractionId}.json`; const written = await this.blobStore.write(bucket, key, payloadJson); const artifact = await this.st0re.createArtifact({ tenant_id: ctx.tenant_id, project_id: ctx.project_id, artifact_type: "document_extraction", app_id: "0pt", produced_by: "0pt.extract_document", object_location: { provider: written.provider, bucket: written.bucket, key: written.key, region: "local" }, content_type: "application/json", size_bytes: written.sizeBytes, checksum_sha256: crypto.createHash("sha256").update(payloadJson).digest("hex"), pii_level: "medium", retention_policy: "document_extraction_standard", data_residency: "eu", risk_level: "L3", correlation_id: ctx.correlation_id, }); record = { ...record, status: "completed", fields, line_items: lineItems, fields_found_count: foundCount, fields_total_count: Object.keys(fields).length, average_confidence: averageConfidence, document_confidence: diResult.documentConfidence, st0re_artifact_id: artifact.artifact_id, updated_at: new Date().toISOString(), }; this.extractions.set(extractionId, record); this.captureService.setStatus(documentId, "extracted"); await this.aud0.ingestEvent({ event_id: `evt_0pt_document_extracted_${extractionId}`, event_type: "0pt.document_extracted", tenant_id: ctx.tenant_id, project_id: ctx.project_id, actor_id: ctx.actor_id, app_id: "0pt", capability_id: "0pt.extract_document", resource_type: "document_extraction", resource_id: extractionId, risk_level: "L3", payload: { document_id: documentId, model_id: ctx.model_id, fields_found_count: foundCount, fields_total_count: Object.keys(fields).length, average_confidence: averageConfidence, st0re_artifact_id: artifact.artifact_id, }, created_at: new Date().toISOString(), correlation_id: ctx.correlation_id, }); await this.aud0.ingestLineageEvent({ lineage_event_id: `lineage_0pt_document_extracted_${extractionId}`, tenant_id: ctx.tenant_id, project_id: ctx.project_id, app_id: "0pt", capability_id: "0pt.extract_document", source_type: "document", source_id: documentId, target_type: "document_extraction", target_id: extractionId, lineage_type: "derived_from", metadata: { model_id: ctx.model_id, st0re_artifact_id: artifact.artifact_id }, created_at: new Date().toISOString(), correlation_id: ctx.correlation_id, }); return record; } catch (e: unknown) { const message = e instanceof Error ? e.message : String(e); record = { ...record, status: "failed", error: message, updated_at: new Date().toISOString() }; this.extractions.set(extractionId, record); try { this.captureService.setStatus(documentId, "extraction_failed"); } catch { /* document may already be gone in edge-case tests — extraction failure record still stands */ } throw e; } } async getExtraction(extractionId: string): Promise { const record = this.extractions.get(extractionId); if (!record) throw new Error(`Document extraction ${extractionId} not found`); return record; } // P014: `tenantId` is optional so existing trusted in-process callers are // unaffected, but the HTTP route now always supplies it -- closing a real // gap: this used to be reachable by documentId alone with NO tenant // check anywhere, so any authenticated caller could list another // tenant's OCR/structuring extraction results (potentially containing // extracted document content, e.g. invoice fields) by guessing or // enumerating documentId. A mismatched tenantId reports identically to // "not found" (no existence oracle), same convention as the rest of the // surface. async listExtractionsForDocument(documentId: string, tenantId?: string): Promise { if (tenantId) { const document = await this.captureService.getDocument(documentId).catch(() => undefined); if (!document || document.tenant_id !== tenantId) { throw new Error(`Document ${documentId} not found`); } } return Array.from(this.extractions.values()).filter((r) => r.document_id === documentId); } } function buildDocumentIntelligenceConfigFromEnv(): DocumentIntelligenceConfig | undefined { const endpoint = process.env.DOC_INTELLIGENCE_ENDPOINT; const apiKey = process.env.DOC_INTELLIGENCE_KEY; if (!endpoint || !apiKey) return undefined; return { endpoint, apiKey, apiVersion: process.env.DOC_INTELLIGENCE_API_VERSION }; } /** Shared singleton (see document-capture-service.ts "Singleton note") wired to the shared capture service. */ export const documentExtractionService = new DocumentExtractionService(documentCaptureService, { documentIntelligence: buildDocumentIntelligenceConfigFromEnv(), });