fix(recovery): restore source removed by production documentation merge
This commit is contained in:
@@ -0,0 +1,343 @@
|
||||
/**
|
||||
* Canonical generic Entity Resolution capability.
|
||||
*
|
||||
* - synchronous candidate generation is deterministic, explainable and
|
||||
* review-aware;
|
||||
* - product hard rules may override probabilities in either direction;
|
||||
* - heavy/statistical ER is delegated to 0pt-motor through N0D;
|
||||
* - the legacy Python production scorer has been removed: Python is now only
|
||||
* an adapter to this canonical decision engine.
|
||||
*/
|
||||
import crypto from "crypto";
|
||||
import { erMatchDecision, erDecisionEnabled, erAutoMatchShadow } from "@siax/0pt-decision";
|
||||
import type { N0dClient, N0dJob } from "../clients/n0d-client";
|
||||
import { SiaxError } from "../platform/errors";
|
||||
import { isResourceRef } from "../platform/resource-ref";
|
||||
|
||||
export interface ErRecord {
|
||||
record_id: string;
|
||||
fields: Record<string, string | number | null | undefined>;
|
||||
}
|
||||
|
||||
export type HardRuleDecision = "force_match" | "force_non_match" | "abstain";
|
||||
|
||||
export interface HardRule {
|
||||
rule_id: string;
|
||||
decide(a: ErRecord, b: ErRecord): HardRuleDecision;
|
||||
}
|
||||
|
||||
export interface ErCandidateRequest {
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
records: ErRecord[];
|
||||
blocking_fields: string[];
|
||||
compare_fields: { field: string; weight?: number }[];
|
||||
auto_match_threshold?: number;
|
||||
review_threshold?: number;
|
||||
correlation_id?: string;
|
||||
}
|
||||
|
||||
export type ErCandidateStatus = "match" | "pending_review" | "non_match";
|
||||
|
||||
export interface ErFieldExplanation {
|
||||
field: string;
|
||||
a: string | null;
|
||||
b: string | null;
|
||||
similarity: number;
|
||||
weight: number;
|
||||
}
|
||||
|
||||
export interface ErCandidatePair {
|
||||
candidate_id: string;
|
||||
record_a: string;
|
||||
record_b: string;
|
||||
score: number;
|
||||
status: ErCandidateStatus;
|
||||
decided_by: string;
|
||||
explanation: ErFieldExplanation[];
|
||||
blocking_key: string;
|
||||
}
|
||||
|
||||
export interface ErCandidateResult {
|
||||
request_id: string;
|
||||
tenant_id: string;
|
||||
pairs: ErCandidatePair[];
|
||||
review_queue: ErCandidatePair[];
|
||||
blocked_comparisons: number;
|
||||
total_possible_comparisons: number;
|
||||
}
|
||||
|
||||
export interface SubmitBatchResolutionInput {
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
records_artifact_ref: string;
|
||||
actor_id?: string;
|
||||
correlation_id?: string;
|
||||
idempotency_key?: string;
|
||||
}
|
||||
|
||||
export class EntityResolutionService {
|
||||
private readonly hardRules: HardRule[] = [];
|
||||
|
||||
constructor(private readonly opts: { n0dClient?: N0dClient } = {}) {}
|
||||
|
||||
registerHardRule(rule: HardRule): void {
|
||||
if (!rule.rule_id?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.er", "hard rule requires rule_id");
|
||||
if (this.hardRules.some((existing) => existing.rule_id === rule.rule_id)) {
|
||||
throw new SiaxError("CONFLICT", "0pt.er", `hard rule ${rule.rule_id} is already registered`);
|
||||
}
|
||||
this.hardRules.push(rule);
|
||||
}
|
||||
|
||||
async generateCandidates(req: ErCandidateRequest): Promise<ErCandidateResult> {
|
||||
if (!req.tenant_id?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.er", "tenant_id is required");
|
||||
if (!req.project_id?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.er", "project_id is required");
|
||||
if (req.records.length < 2 || req.records.length > 5_000) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", "records must contain between 2 and 5000 items");
|
||||
}
|
||||
if (new Set(req.records.map((record) => record.record_id)).size !== req.records.length) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", "record_id values must be unique");
|
||||
}
|
||||
if (req.blocking_fields.length < 1 || req.blocking_fields.length > 20) {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.er",
|
||||
"blocking_fields must contain 1-20 fields — unblocked all-pairs comparison is refused",
|
||||
);
|
||||
}
|
||||
if (req.compare_fields.length < 1 || req.compare_fields.length > 50) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", "compare_fields must contain 1-50 fields");
|
||||
}
|
||||
for (const { field, weight = 1 } of req.compare_fields) {
|
||||
if (!field?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.er", "compare field name is required");
|
||||
if (!Number.isFinite(weight) || weight < 0 || weight > 100) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", `invalid weight for compare field ${field}`);
|
||||
}
|
||||
}
|
||||
|
||||
const autoThreshold = req.auto_match_threshold ?? 0.9;
|
||||
const reviewThreshold = req.review_threshold ?? 0.6;
|
||||
if (
|
||||
!Number.isFinite(autoThreshold) || autoThreshold < 0 || autoThreshold > 1 ||
|
||||
!Number.isFinite(reviewThreshold) || reviewThreshold < 0 || reviewThreshold > 1 ||
|
||||
reviewThreshold >= autoThreshold
|
||||
) {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.er",
|
||||
"thresholds must be within 0..1 and review_threshold must be lower than auto_match_threshold",
|
||||
);
|
||||
}
|
||||
|
||||
const blocks = new Map<string, ErRecord[]>();
|
||||
for (const rec of req.records) {
|
||||
if (!rec.record_id?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.er", "every record requires record_id");
|
||||
const key = req.blocking_fields.map((field) => normalize(String(rec.fields[field] ?? ""))).join("|");
|
||||
const bucket = blocks.get(key) ?? [];
|
||||
bucket.push(rec);
|
||||
blocks.set(key, bucket);
|
||||
}
|
||||
|
||||
const pairs: ErCandidatePair[] = [];
|
||||
let blockedComparisons = 0;
|
||||
for (const [key, bucket] of blocks) {
|
||||
if (key.replace(/\|/g, "") === "") continue;
|
||||
for (let i = 0; i < bucket.length; i++) {
|
||||
for (let j = i + 1; j < bucket.length; j++) {
|
||||
blockedComparisons += 1;
|
||||
pairs.push(await this.comparePair(bucket[i], bucket[j], req, autoThreshold, reviewThreshold, key));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const n = req.records.length;
|
||||
return {
|
||||
request_id: `errq_${crypto.randomUUID()}`,
|
||||
tenant_id: req.tenant_id,
|
||||
pairs,
|
||||
review_queue: pairs.filter((pair) => pair.status === "pending_review"),
|
||||
blocked_comparisons: blockedComparisons,
|
||||
total_possible_comparisons: (n * (n - 1)) / 2,
|
||||
};
|
||||
}
|
||||
|
||||
/** Heavy/statistical path — an idempotent N0D job executed by 0pt-motor. */
|
||||
async submitBatchResolution(input: SubmitBatchResolutionInput): Promise<N0dJob> {
|
||||
if (!input.tenant_id?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.er", "tenant_id is required");
|
||||
if (!input.project_id?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.er", "project_id is required");
|
||||
if (!isResourceRef(input.records_artifact_ref, "st0re", "artifact")) {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.er",
|
||||
"records_artifact_ref must be a canonical st0re:artifact:... reference",
|
||||
);
|
||||
}
|
||||
if (!this.opts.n0dClient) {
|
||||
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "n0d", "no N0D client configured for batch ER");
|
||||
}
|
||||
const idempotencyKey = input.idempotency_key?.trim() || crypto
|
||||
.createHash("sha256")
|
||||
.update(`0pt.er.batch.v1|${input.tenant_id}|${input.project_id}|${input.records_artifact_ref}`)
|
||||
.digest("hex");
|
||||
|
||||
return this.opts.n0dClient.createJob({
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
job_type: "0pt.er.batch_resolution",
|
||||
capability_id: "0pt.er.batch.v1",
|
||||
requested_by_app_id: "0pt",
|
||||
requested_by_actor_id: input.actor_id ?? "svc_0pt",
|
||||
input_artifact_ids: [input.records_artifact_ref],
|
||||
output_artifact_type: "er_candidate_set",
|
||||
requires_backbone: false,
|
||||
requires_gpu: false,
|
||||
required_node_roles: ["compute_general"],
|
||||
resource_profile: "large",
|
||||
risk_level: "L3",
|
||||
correlation_id: input.correlation_id,
|
||||
idempotency_key: idempotencyKey,
|
||||
timeout_seconds: 3_600,
|
||||
max_retries: 2,
|
||||
});
|
||||
}
|
||||
|
||||
private async comparePair(
|
||||
a: ErRecord,
|
||||
b: ErRecord,
|
||||
req: ErCandidateRequest,
|
||||
autoThreshold: number,
|
||||
reviewThreshold: number,
|
||||
blockingKey: string,
|
||||
): Promise<ErCandidatePair> {
|
||||
for (const rule of this.hardRules) {
|
||||
const decision = rule.decide(a, b);
|
||||
if (decision !== "abstain") {
|
||||
return {
|
||||
candidate_id: `erc_${crypto.randomUUID()}`,
|
||||
record_a: a.record_id,
|
||||
record_b: b.record_id,
|
||||
score: decision === "force_match" ? 1 : 0,
|
||||
status: decision === "force_match" ? "match" : "non_match",
|
||||
decided_by: rule.rule_id,
|
||||
explanation: this.explain(a, b, req),
|
||||
blocking_key: blockingKey,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const explanation = this.explain(a, b, req);
|
||||
const weightSum = explanation.reduce((sum, item) => sum + item.weight, 0);
|
||||
const score = weightSum === 0
|
||||
? 0
|
||||
: explanation.reduce((sum, item) => sum + item.similarity * item.weight, 0) / weightSum;
|
||||
const rounded = Math.round(score * 1000) / 1000;
|
||||
let status: ErCandidateStatus = rounded >= autoThreshold
|
||||
? "match"
|
||||
: rounded >= reviewThreshold
|
||||
? "pending_review"
|
||||
: "non_match";
|
||||
// DECISION-plane (D1 advisory): borderline-band → decision.match tie-break.
|
||||
// Deterministisk status GÄLLER om decision abstainar (doctor DEC-*:
|
||||
// decision informerar, ersätter aldrig D0-authority).
|
||||
if (erDecisionEnabled() && Math.abs(rounded - reviewThreshold) <= DECISION_BORDERLINE_BAND) {
|
||||
const m = await erMatchDecision({
|
||||
record_a: a.record_id,
|
||||
record_b: b.record_id,
|
||||
similarities: explanation.map((item) => ({ field: item.field, similarity: item.similarity, weight: item.weight })),
|
||||
});
|
||||
if (m.decided && m.label) status = m.label as ErCandidateStatus;
|
||||
}
|
||||
// Shadow-eval (Jev-everywhere): D0 auto-match förblir auktoritet —
|
||||
// kvaliteten mäts fire-and-forget mot beslutsplanet (0PT_DECISION_ER_SHADOW,
|
||||
// OFF by default). Hängande/felande fetch påverkar ALDRIG svaret.
|
||||
if (rounded >= autoThreshold) {
|
||||
void erAutoMatchShadow({
|
||||
tenant_id: req.tenant_id,
|
||||
target: { record_id: a.record_id, fields: a.fields },
|
||||
candidates: [{ record_id: b.record_id, fields: b.fields }],
|
||||
score: rounded,
|
||||
}).catch(() => null);
|
||||
}
|
||||
return {
|
||||
candidate_id: `erc_${crypto.randomUUID()}`,
|
||||
record_a: a.record_id,
|
||||
record_b: b.record_id,
|
||||
score: rounded,
|
||||
status,
|
||||
decided_by: "score",
|
||||
explanation,
|
||||
blocking_key: blockingKey,
|
||||
};
|
||||
}
|
||||
|
||||
private explain(a: ErRecord, b: ErRecord, req: ErCandidateRequest): ErFieldExplanation[] {
|
||||
return req.compare_fields.map(({ field, weight }) => {
|
||||
const av = a.fields[field] === null || a.fields[field] === undefined ? null : String(a.fields[field]);
|
||||
const bv = b.fields[field] === null || b.fields[field] === undefined ? null : String(b.fields[field]);
|
||||
return {
|
||||
field,
|
||||
a: av,
|
||||
b: bv,
|
||||
similarity: similarity(av, bv),
|
||||
weight: weight ?? 1,
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function normalize(s: string): string {
|
||||
return s
|
||||
.toLowerCase()
|
||||
.replace(/ø/g, "o")
|
||||
.normalize("NFKD")
|
||||
.replace(/[̀-ͯ]/g, "")
|
||||
.replace(/[^a-z0-9 ]/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function similarity(a: string | null, b: string | null): number {
|
||||
if (a === null || b === null) return 0;
|
||||
const na = normalize(a);
|
||||
const nb = normalize(b);
|
||||
if (na === "" && nb === "") return 0;
|
||||
if (na === nb) return 1;
|
||||
return Math.max(tokenJaccard(na, nb), 1 - normalizedLevenshtein(na, nb));
|
||||
}
|
||||
|
||||
function tokenJaccard(a: string, b: string): number {
|
||||
const ta = new Set(a.split(" "));
|
||||
const tb = new Set(b.split(" "));
|
||||
const intersection = [...ta].filter((token) => tb.has(token)).length;
|
||||
const union = new Set([...ta, ...tb]).size;
|
||||
return union === 0 ? 0 : intersection / union;
|
||||
}
|
||||
|
||||
function normalizedLevenshtein(a: string, b: string): number {
|
||||
const maxLen = Math.max(a.length, b.length);
|
||||
if (maxLen === 0) return 0;
|
||||
return levenshtein(a, b) / maxLen;
|
||||
}
|
||||
|
||||
function levenshtein(a: string, b: string): number {
|
||||
const m = a.length;
|
||||
const n = b.length;
|
||||
if (m === 0) return n;
|
||||
if (n === 0) return m;
|
||||
let prev = new Array<number>(n + 1);
|
||||
let curr = new Array<number>(n + 1);
|
||||
for (let j = 0; j <= n; j++) prev[j] = j;
|
||||
for (let i = 1; i <= m; i++) {
|
||||
curr[0] = i;
|
||||
for (let j = 1; j <= n; j++) {
|
||||
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
||||
curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost);
|
||||
}
|
||||
[prev, curr] = [curr, prev];
|
||||
}
|
||||
return prev[n];
|
||||
}
|
||||
|
||||
|
||||
// --- DECISION-plane wiring (D1 advisory, aldrig gate) -------------------
|
||||
const DECISION_BORDERLINE_BAND = Number(process.env.DECISION_BORDERLINE_BAND ?? 0.05);
|
||||
@@ -0,0 +1,110 @@
|
||||
import { SiaxError } from "../platform/errors";
|
||||
import type { ErCandidateRequest, ErRecord } from "./entity-resolution-service";
|
||||
|
||||
function finiteThreshold(value: unknown, field: string, fallback: number): number {
|
||||
if (value === undefined) return fallback;
|
||||
const parsed = Number(value);
|
||||
if (!Number.isFinite(parsed) || parsed < 0 || parsed > 1) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", `${field} must be a finite number between 0 and 1`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export function parseErCandidateRequest(raw: unknown): ErCandidateRequest {
|
||||
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", "JSON object body is required");
|
||||
}
|
||||
const body = raw as Record<string, unknown>;
|
||||
const tenantId = typeof body.tenant_id === "string" ? body.tenant_id.trim() : "";
|
||||
const projectId = typeof body.project_id === "string" ? body.project_id.trim() : "";
|
||||
if (!tenantId) throw new SiaxError("INVALID_REQUEST", "0pt.er", "tenant_id is required");
|
||||
if (!projectId) throw new SiaxError("INVALID_REQUEST", "0pt.er", "project_id is required");
|
||||
|
||||
if (!Array.isArray(body.records) || body.records.length < 2 || body.records.length > 5_000) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", "records must contain between 2 and 5000 items");
|
||||
}
|
||||
if (
|
||||
!Array.isArray(body.blocking_fields) ||
|
||||
body.blocking_fields.length === 0 ||
|
||||
body.blocking_fields.length > 20 ||
|
||||
!body.blocking_fields.every((v) => typeof v === "string" && v.trim().length > 0)
|
||||
) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", "blocking_fields must contain 1-20 non-empty strings");
|
||||
}
|
||||
if (!Array.isArray(body.compare_fields) || body.compare_fields.length === 0 || body.compare_fields.length > 50) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", "compare_fields must contain 1-50 items");
|
||||
}
|
||||
|
||||
const records: ErRecord[] = body.records.map((value, index) => {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", `records[${index}] must be an object`);
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
if (typeof record.record_id !== "string" || !record.record_id.trim()) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", `records[${index}].record_id is required`);
|
||||
}
|
||||
if (!record.fields || typeof record.fields !== "object" || Array.isArray(record.fields)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", `records[${index}].fields must be an object`);
|
||||
}
|
||||
const fields = record.fields as Record<string, unknown>;
|
||||
if (Object.keys(fields).length > 200) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", `records[${index}].fields exceeds 200 fields`);
|
||||
}
|
||||
const normalizedFields: ErRecord["fields"] = {};
|
||||
for (const [field, fieldValue] of Object.entries(fields)) {
|
||||
if (!field.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.er", `records[${index}] contains an empty field name`);
|
||||
if (
|
||||
fieldValue !== null &&
|
||||
fieldValue !== undefined &&
|
||||
typeof fieldValue !== "string" &&
|
||||
typeof fieldValue !== "number"
|
||||
) {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.er",
|
||||
`records[${index}].fields.${field} must be string, number, null or undefined`,
|
||||
);
|
||||
}
|
||||
normalizedFields[field.trim()] = fieldValue as string | number | null | undefined;
|
||||
}
|
||||
return { record_id: record.record_id.trim(), fields: normalizedFields };
|
||||
});
|
||||
|
||||
if (new Set(records.map((record) => record.record_id)).size !== records.length) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", "record_id values must be unique within the request");
|
||||
}
|
||||
|
||||
const compareFields = body.compare_fields.map((value, index) => {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", `compare_fields[${index}] must be an object`);
|
||||
}
|
||||
const item = value as Record<string, unknown>;
|
||||
if (typeof item.field !== "string" || !item.field.trim()) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", `compare_fields[${index}].field is required`);
|
||||
}
|
||||
const weight = item.weight === undefined ? undefined : Number(item.weight);
|
||||
if (weight !== undefined && (!Number.isFinite(weight) || weight < 0 || weight > 100)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", `compare_fields[${index}].weight must be between 0 and 100`);
|
||||
}
|
||||
return { field: item.field.trim(), ...(weight !== undefined ? { weight } : {}) };
|
||||
});
|
||||
|
||||
const autoThreshold = finiteThreshold(body.auto_match_threshold, "auto_match_threshold", 0.9);
|
||||
const reviewThreshold = finiteThreshold(body.review_threshold, "review_threshold", 0.6);
|
||||
if (reviewThreshold >= autoThreshold) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", "review_threshold must be lower than auto_match_threshold");
|
||||
}
|
||||
|
||||
return {
|
||||
tenant_id: tenantId,
|
||||
project_id: projectId,
|
||||
records,
|
||||
blocking_fields: (body.blocking_fields as string[]).map((value) => value.trim()),
|
||||
compare_fields: compareFields,
|
||||
auto_match_threshold: autoThreshold,
|
||||
review_threshold: reviewThreshold,
|
||||
...(typeof body.correlation_id === "string" && body.correlation_id.trim()
|
||||
? { correlation_id: body.correlation_id.trim() }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user