Files
0pt/src/metadata/openmetadata-adapter.ts
T
Simon AxelssonandClaude Fable 5 b46807da75 feat(data-os): P0 capability layer — source registry, search, quality, dataset versions, metadata, analytics, features, ER
- Source Registry (§10): rights/license/purpose metadata, provenance,
  freshness SLA, health checks, describe/query; purpose gate FAILS CLOSED
  (no rights record => deny); cross-tenant probes => NOT_FOUND.
- Search (§10/§42): OpenSearch REST adapter (no SDK) + SearchService with
  MANDATORY tenant + purpose/rights filtering, versioned index definitions,
  bulk, generation-per-rebuild (opt-{name}--g{n}) with atomic alias cutover,
  rollback, failed-rebuild generation discard. OpenSearch never canonical.
- Quality (§10): versioned rule registry (GE-JSON interchange), local
  engine for the GE core subset (never silently passes unsupported types),
  runs/results/issues/score/trend, whole-run delegation to the GE backend
  via N0D (0pt.quality.ge_run) - never fabricated results.
- Dataset (§10/§5/§53): immutable numbered versions holding canonical
  st0re:artifact refs only, idempotent createVersion, export registers the
  ST0RE artifact + AUD0 evidence + OpenLineage.
- Metadata (§10): OpenMetadata PUT /api/v1/tables projection adapter +
  projectAll rebuild path; projection never canonical.
- Analytics (§10): ClickHouse HTTP adapter, closed metric-query DSL with
  FORCED tenant predicate + readonly/max_execution_time/max_result_rows,
  TTL retention. Zero connection to b0ts/quarantined pipelines.
- Features (§10): versioned feature-view registry, real Feast
  get-online-features adapter, materialize/get-historical as N0D delegation.
- ER (§10): generic candidate API with deterministic blocking + transparent
  scoring + per-field explanations, product hard-rule hook (force both
  directions), review-band handoff, Splink batch via N0D contract.

93 new unit/contract tests; suite 248 passed / 1 skipped.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-31 16:49:32 +02:00

135 lines
5.0 KiB
TypeScript

/**
* src/metadata/openmetadata-adapter.ts
* =====================================
* OpenMetadata PROJECTION adapter (Master-DoD §10 Metadata: "OpenMetadata
* som projection (inte canonical)").
*
* Doctrine, stated plainly:
* - 0pt's registries (DatasetService/DatasetVersionService/SourceRegistry)
* are CANONICAL. OpenMetadata is a rebuildable projection for humans —
* deleting the OpenMetadata instance loses NOTHING canonical.
* - `projectAll()` therefore exists and is the recovery path (§42).
*
* Wire contract: OpenMetadata 1.x REST — PUT /api/v1/tables (createOrUpdate
* semantics) with a bearer bot token (OM's standard ingestion-bot auth).
* The subset used (fullyQualifiedName via service/database/schema hierarchy,
* columns, description, tags) is stable across OM 1.x.
*
* STATUS HONESTY: adapter + projection service are real code with contract
* tests against a recording double. NOT yet integration-tested against a
* live OpenMetadata (no local container in this pass) — classified
* IMPLEMENTED, not INTEGRATION_TESTED, in STATUS.md.
*/
import { SiaxError, siaxErrorCodeForUpstreamStatus } from "../platform/errors";
import type { Dataset } from "../types/dataset";
import type { DatasetVersion } from "../services/dataset-version-service";
export interface OpenMetadataAdapterOptions {
baseUrl: string;
/** OM bot JWT. Comes from env/secret ref — never hardcoded. */
token?: string;
timeoutMs?: number;
}
/** The projected table entity (OM createOrUpdate subset). */
export interface OmTableUpsert {
name: string;
databaseSchema: string;
description?: string;
columns: { name: string; dataType: string; description?: string }[];
tags?: { tagFQN: string }[];
}
export interface MetadataProjectionResult {
projected: number;
failures: { dataset_id: string; error: string }[];
}
export class OpenMetadataAdapter {
private readonly baseUrl: string;
private readonly token?: string;
private readonly timeoutMs: number;
constructor(opts: OpenMetadataAdapterOptions) {
this.baseUrl = opts.baseUrl.replace(/\/+$/, "");
this.token = opts.token;
this.timeoutMs = opts.timeoutMs ?? 15_000;
}
async upsertTable(table: OmTableUpsert): Promise<void> {
const headers: Record<string, string> = { "Content-Type": "application/json" };
if (this.token) headers["Authorization"] = `Bearer ${this.token}`;
let res: Response;
try {
res = await fetch(`${this.baseUrl}/api/v1/tables`, {
method: "PUT",
headers,
body: JSON.stringify(table),
signal: AbortSignal.timeout(this.timeoutMs),
});
} catch (cause) {
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "openmetadata", "PUT /api/v1/tables unreachable", { cause });
}
if (!res.ok) {
throw new SiaxError(
siaxErrorCodeForUpstreamStatus(res.status),
"openmetadata",
`PUT /api/v1/tables → HTTP ${res.status}`
);
}
}
}
/**
* Projects 0pt's canonical dataset registry into OpenMetadata. Pure mapping +
* adapter calls; failures are collected per dataset, never silently dropped.
*/
export class MetadataProjectionService {
constructor(
private readonly adapter: OpenMetadataAdapter,
/** OM schema FQN the projection writes under, e.g. "0pt.default.datasets". */
private readonly schemaFqn: string = "0pt.default.datasets"
) {}
toTableUpsert(dataset: Dataset, latestVersion?: DatasetVersion): OmTableUpsert {
return {
name: dataset.dataset_id,
databaseSchema: this.schemaFqn,
description:
`${dataset.name} — PROJECTION of canonical 0pt dataset registry entry ` +
`0pt:dataset:${dataset.dataset_id}` +
(latestVersion ? ` (latest version ${latestVersion.version}, schema ${latestVersion.schema_version})` : "") +
`. Canonical truth lives in 0pt; this record is rebuildable.`,
columns: [
{ name: "dataset_id", dataType: "STRING", description: "Canonical 0pt dataset id" },
{ name: "tenant_id", dataType: "STRING", description: "Owning tenant" },
{ name: "dataset_type", dataType: "STRING", description: dataset.dataset_type },
{ name: "status", dataType: "STRING", description: dataset.status },
],
tags: [{ tagFQN: "0ptProjection.canonical-elsewhere" }],
};
}
async projectDataset(dataset: Dataset, latestVersion?: DatasetVersion): Promise<void> {
await this.adapter.upsertTable(this.toTableUpsert(dataset, latestVersion));
}
/** The §42 recovery path: re-project everything from canonical registries. */
async projectAll(
datasets: Dataset[],
latestVersions: Map<string, DatasetVersion>
): Promise<MetadataProjectionResult> {
const failures: { dataset_id: string; error: string }[] = [];
let projected = 0;
for (const ds of datasets) {
try {
await this.projectDataset(ds, latestVersions.get(ds.dataset_id));
projected++;
} catch (err) {
failures.push({ dataset_id: ds.dataset_id, error: err instanceof Error ? err.message : String(err) });
}
}
return { projected, failures };
}
}