Files
0pt/src/services/document-extraction-service.ts
T
admin bd1c20edb2
CI (SIAX Cloud) / integration (push) Skipped
CI (SIAX Cloud) / sonar (push) Skipped
masterplan-lock / check (push) Successful in 5s
Security Scan (SIAX Cloud) / security (push) Successful in 15s
CI (SIAX Cloud) / security (push) Successful in 16s
CI (SIAX Cloud) / sonar (pull_request) Skipped
masterplan-lock / check (pull_request) Successful in 7s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 21s
CI (SIAX Cloud) / security (pull_request) Successful in 25s
CI (SIAX Cloud) / quality (push) Successful in 1m17s
CI (SIAX Cloud) / integration (pull_request) Successful in 54s
CI (SIAX Cloud) / quality (pull_request) Successful in 1m7s
feat: add guarded document workbench foundations
2026-11-02 01:26:36 +01:00

291 lines
12 KiB
TypeScript

/**
* src/services/document-extraction-service.ts
* ==============================================
* Evidence-based field extraction for a captured document.
*
* Calls DocumentIntelligenceAdapter and NEVER fabricates a value: a field
* key is only populated with a non-null value when Document Intelligence
* actually returned one, and it always carries DI's own confidence score
* (or null when DI reported none). Fields DI did not find are left blank —
* { value: null, confidence: <DI's confidence if any>, found: false } —
* never defaulted, guessed, or backfilled from anything else. This mirrors
* the evidence-based discipline in
* v0-datahub/lib/invoice-inbox/doc-intelligence.ts exactly (it never
* invented an amount/vendor/date that DI didn't return); it is a
* correctness property, not a style preference, so it is enforced twice —
* once inside the adapter's field normalizer, and again defensively here.
*
* modelId is caller-supplied ("prebuilt-invoice", "prebuilt-receipt",
* "prebuilt-layout", or a custom model id) — this service does not assume
* any particular document type. Invoice-specific reasoning (vendor
* matching, dup-hash, DQ scoring, invoice classification) was intentionally
* left in v0-datahub/lib/invoice-inbox and is NOT ported here.
*/
import crypto from "crypto";
import {
DocumentIntelligenceAdapter,
type DocumentIntelligenceConfig,
type ExtractedField as AdapterField,
} from "../adapters/document-intelligence-adapter";
import { LocalBlobStore } from "../adapters/local-blob-store";
import { createCl0udClient, type Cl0udClient } from "../clients/cl0ud-client";
import { createSt0reClient, type St0reClient } from "../clients/st0re-client";
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import { DocumentCaptureService, documentCaptureService } from "./document-capture-service";
import type { DocumentExtractionRecord, ExtractedField } from "../types/document-extraction";
export interface ExtractDocumentContext {
tenant_id: string;
project_id: string;
actor_id: string;
correlation_id: string;
/** e.g. "prebuilt-invoice" | "prebuilt-receipt" | "prebuilt-layout" | a custom model id. */
model_id: string;
}
/**
* Defensive re-application of the "never fabricate" rule at the service
* boundary. Even though the adapter's normalizeField() already enforces
* this, a field is only ever considered found here if it is BOTH marked
* found by the adapter AND carries a genuinely non-empty value — so a
* malformed/partial adapter response can never leak a fabricated value
* through this service.
*/
function toEvidenceFields(fields: Record<string, AdapterField>): Record<string, ExtractedField> {
const out: Record<string, ExtractedField> = {};
for (const [key, f] of Object.entries(fields)) {
const found = !!f?.found && f.value !== null && f.value !== undefined && f.value !== "";
out[key] = { value: found ? f.value : null, confidence: f?.confidence ?? null, found };
}
return out;
}
export class DocumentExtractionService {
private extractions: Map<string, DocumentExtractionRecord> = new Map();
private cl0ud: Cl0udClient;
private st0re: St0reClient;
private aud0: Aud0Client;
private blobStore: LocalBlobStore;
private di: DocumentIntelligenceAdapter | null;
constructor(
private captureService: DocumentCaptureService,
opts?: {
cl0udBaseUrl?: string;
st0reBaseUrl?: string;
aud0BaseUrl?: string;
documentIntelligence?: DocumentIntelligenceConfig;
blobStore?: LocalBlobStore;
},
) {
this.cl0ud = createCl0udClient(opts?.cl0udBaseUrl);
this.st0re = createSt0reClient(opts?.st0reBaseUrl);
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.blobStore = opts?.blobStore ?? new LocalBlobStore();
this.di =
opts?.documentIntelligence && DocumentIntelligenceAdapter.isConfigured(opts.documentIntelligence)
? new DocumentIntelligenceAdapter(opts.documentIntelligence)
: null;
}
async extractDocument(documentId: string, ctx: ExtractDocumentContext): Promise<DocumentExtractionRecord> {
if (!this.di) {
throw new Error(
"DocumentExtractionService: Document Intelligence is not configured (missing endpoint/apiKey) — refusing to fabricate a result",
);
}
if (!ctx.model_id) {
throw new Error("extractDocument: missing model_id");
}
if (!ctx.tenant_id || !ctx.project_id || !ctx.actor_id) {
throw new Error("extractDocument: missing tenant_id/project_id/actor_id");
}
const document = await this.captureService.getDocument(documentId);
// The caller's verified tenant alone is insufficient: the document being
// read must belong to the same tenant AND project before policy or bytes.
if (document.tenant_id !== ctx.tenant_id || document.project_id !== ctx.project_id) {
throw new Error(`Document ${documentId} not found`);
}
const policy = await this.cl0ud.evaluatePolicy({
tenant_id: ctx.tenant_id,
project_id: ctx.project_id,
actor_id: ctx.actor_id,
capability_id: "0pt.extract_document",
resource_type: "document",
resource_id: documentId,
risk_level: "L3",
correlation_id: ctx.correlation_id,
});
if (policy.decision === "deny") throw new Error(`Policy denied: ${policy.reason}`);
if (policy.decision === "approval_required") throw new Error(`Policy requires approval: ${policy.reason}`);
const extractionId = `dex_${crypto.randomUUID()}`;
const now = new Date().toISOString();
let record: DocumentExtractionRecord = {
extraction_id: extractionId,
document_id: documentId,
tenant_id: ctx.tenant_id,
project_id: ctx.project_id,
model_id: ctx.model_id,
status: "requested",
fields: {},
line_items: [],
fields_found_count: 0,
fields_total_count: 0,
average_confidence: null,
document_confidence: null,
correlation_id: ctx.correlation_id,
created_at: now,
};
this.extractions.set(extractionId, record);
try {
const bytes = await this.captureService.getDocumentBytes(documentId);
const diResult = await this.di.analyzeDocument(bytes, ctx.model_id);
const fields = toEvidenceFields(diResult.fields);
const lineItems = Object.values(diResult.arrayFields).flatMap((items) => items.map((item) => toEvidenceFields(item)));
const foundCount = Object.values(fields).filter((f) => f.found).length;
// Average only over FOUND fields — a not-found field's confidence (e.g. "31% sure this
// region isn't a date after all") describes DI's certainty about absence, not a value,
// and must not drag down the confidence of the values that were actually extracted.
const confidences = Object.values(fields)
.filter((f) => f.found)
.map((f) => f.confidence)
.filter((c): c is number => c != null);
const averageConfidence = confidences.length ? confidences.reduce((a, b) => a + b, 0) / confidences.length : null;
// Persist the extraction result as an artifact too — same interim-storage gap as
// document-capture-service.ts (see that file's header for the full explanation).
const payloadJson = Buffer.from(
JSON.stringify({ model_id: ctx.model_id, fields, line_items: lineItems, content: diResult.content }),
"utf8",
);
const bucket = "0pt-document-extractions";
const key = `${ctx.tenant_id}/${ctx.project_id}/${documentId}/${extractionId}.json`;
const written = await this.blobStore.write(bucket, key, payloadJson);
const artifact = await this.st0re.createArtifact({
tenant_id: ctx.tenant_id,
project_id: ctx.project_id,
artifact_type: "document_extraction",
app_id: "0pt",
produced_by: "0pt.extract_document",
object_location: { provider: written.provider, bucket: written.bucket, key: written.key, region: "local" },
content_type: "application/json",
size_bytes: written.sizeBytes,
checksum_sha256: crypto.createHash("sha256").update(payloadJson).digest("hex"),
pii_level: "high",
retention_policy: "document_extraction_standard",
data_residency: "eu",
risk_level: "L3",
correlation_id: ctx.correlation_id,
});
record = {
...record,
status: "completed",
fields,
line_items: lineItems,
fields_found_count: foundCount,
fields_total_count: Object.keys(fields).length,
average_confidence: averageConfidence,
document_confidence: diResult.documentConfidence,
st0re_artifact_id: artifact.artifact_id,
updated_at: new Date().toISOString(),
};
this.extractions.set(extractionId, record);
this.captureService.setStatus(documentId, "extracted");
await this.aud0.ingestEvent({
event_id: `evt_0pt_document_extracted_${extractionId}`,
event_type: "0pt.document_extracted",
tenant_id: ctx.tenant_id,
project_id: ctx.project_id,
actor_id: ctx.actor_id,
app_id: "0pt",
capability_id: "0pt.extract_document",
resource_type: "document_extraction",
resource_id: extractionId,
risk_level: "L3",
payload: {
document_id: documentId,
model_id: ctx.model_id,
fields_found_count: foundCount,
fields_total_count: Object.keys(fields).length,
average_confidence: averageConfidence,
st0re_artifact_id: artifact.artifact_id,
},
created_at: new Date().toISOString(),
correlation_id: ctx.correlation_id,
});
await this.aud0.ingestLineageEvent({
lineage_event_id: `lineage_0pt_document_extracted_${extractionId}`,
tenant_id: ctx.tenant_id,
project_id: ctx.project_id,
app_id: "0pt",
capability_id: "0pt.extract_document",
source_type: "document",
source_id: documentId,
target_type: "document_extraction",
target_id: extractionId,
lineage_type: "derived_from",
metadata: { model_id: ctx.model_id, st0re_artifact_id: artifact.artifact_id },
created_at: new Date().toISOString(),
correlation_id: ctx.correlation_id,
});
return record;
} catch (e: unknown) {
const message = e instanceof Error ? e.message : String(e);
record = { ...record, status: "failed", error: message, updated_at: new Date().toISOString() };
this.extractions.set(extractionId, record);
try {
this.captureService.setStatus(documentId, "extraction_failed");
} catch {
/* document may already be gone in edge-case tests — extraction failure record still stands */
}
throw e;
}
}
async getExtraction(extractionId: string): Promise<DocumentExtractionRecord> {
const record = this.extractions.get(extractionId);
if (!record) throw new Error(`Document extraction ${extractionId} not found`);
return record;
}
// P014: `tenantId` is optional so existing trusted in-process callers are
// unaffected, but the HTTP route now always supplies it -- closing a real
// gap: this used to be reachable by documentId alone with NO tenant
// check anywhere, so any authenticated caller could list another
// tenant's OCR/structuring extraction results (potentially containing
// extracted document content, e.g. invoice fields) by guessing or
// enumerating documentId. A mismatched tenantId reports identically to
// "not found" (no existence oracle), same convention as the rest of the
// surface.
async listExtractionsForDocument(documentId: string, tenantId?: string): Promise<DocumentExtractionRecord[]> {
if (tenantId) {
const document = await this.captureService.getDocument(documentId).catch(() => undefined);
if (!document || document.tenant_id !== tenantId) {
throw new Error(`Document ${documentId} not found`);
}
}
return Array.from(this.extractions.values()).filter((r) => r.document_id === documentId);
}
}
function buildDocumentIntelligenceConfigFromEnv(): DocumentIntelligenceConfig | undefined {
const endpoint = process.env.DOC_INTELLIGENCE_ENDPOINT;
const apiKey = process.env.DOC_INTELLIGENCE_KEY;
if (!endpoint || !apiKey) return undefined;
return { endpoint, apiKey, apiVersion: process.env.DOC_INTELLIGENCE_API_VERSION };
}
/** Shared singleton (see document-capture-service.ts "Singleton note") wired to the shared capture service. */
export const documentExtractionService = new DocumentExtractionService(documentCaptureService, {
documentIntelligence: buildDocumentIntelligenceConfigFromEnv(),
});