- Source Registry (§10): rights/license/purpose metadata, provenance,
freshness SLA, health checks, describe/query; purpose gate FAILS CLOSED
(no rights record => deny); cross-tenant probes => NOT_FOUND.
- Search (§10/§42): OpenSearch REST adapter (no SDK) + SearchService with
MANDATORY tenant + purpose/rights filtering, versioned index definitions,
bulk, generation-per-rebuild (opt-{name}--g{n}) with atomic alias cutover,
rollback, failed-rebuild generation discard. OpenSearch never canonical.
- Quality (§10): versioned rule registry (GE-JSON interchange), local
engine for the GE core subset (never silently passes unsupported types),
runs/results/issues/score/trend, whole-run delegation to the GE backend
via N0D (0pt.quality.ge_run) - never fabricated results.
- Dataset (§10/§5/§53): immutable numbered versions holding canonical
st0re:artifact refs only, idempotent createVersion, export registers the
ST0RE artifact + AUD0 evidence + OpenLineage.
- Metadata (§10): OpenMetadata PUT /api/v1/tables projection adapter +
projectAll rebuild path; projection never canonical.
- Analytics (§10): ClickHouse HTTP adapter, closed metric-query DSL with
FORCED tenant predicate + readonly/max_execution_time/max_result_rows,
TTL retention. Zero connection to b0ts/quarantined pipelines.
- Features (§10): versioned feature-view registry, real Feast
get-online-features adapter, materialize/get-historical as N0D delegation.
- ER (§10): generic candidate API with deterministic blocking + transparent
scoring + per-field explanations, product hard-rule hook (force both
directions), review-band handoff, Splink batch via N0D contract.
93 new unit/contract tests; suite 248 passed / 1 skipped.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
135 lines
5.0 KiB
TypeScript
135 lines
5.0 KiB
TypeScript
/**
|
|
* src/metadata/openmetadata-adapter.ts
|
|
* =====================================
|
|
* OpenMetadata PROJECTION adapter (Master-DoD §10 Metadata: "OpenMetadata
|
|
* som projection (inte canonical)").
|
|
*
|
|
* Doctrine, stated plainly:
|
|
* - 0pt's registries (DatasetService/DatasetVersionService/SourceRegistry)
|
|
* are CANONICAL. OpenMetadata is a rebuildable projection for humans —
|
|
* deleting the OpenMetadata instance loses NOTHING canonical.
|
|
* - `projectAll()` therefore exists and is the recovery path (§42).
|
|
*
|
|
* Wire contract: OpenMetadata 1.x REST — PUT /api/v1/tables (createOrUpdate
|
|
* semantics) with a bearer bot token (OM's standard ingestion-bot auth).
|
|
* The subset used (fullyQualifiedName via service/database/schema hierarchy,
|
|
* columns, description, tags) is stable across OM 1.x.
|
|
*
|
|
* STATUS HONESTY: adapter + projection service are real code with contract
|
|
* tests against a recording double. NOT yet integration-tested against a
|
|
* live OpenMetadata (no local container in this pass) — classified
|
|
* IMPLEMENTED, not INTEGRATION_TESTED, in STATUS.md.
|
|
*/
|
|
import { SiaxError, siaxErrorCodeForUpstreamStatus } from "../platform/errors";
|
|
import type { Dataset } from "../types/dataset";
|
|
import type { DatasetVersion } from "../services/dataset-version-service";
|
|
|
|
export interface OpenMetadataAdapterOptions {
|
|
baseUrl: string;
|
|
/** OM bot JWT. Comes from env/secret ref — never hardcoded. */
|
|
token?: string;
|
|
timeoutMs?: number;
|
|
}
|
|
|
|
/** The projected table entity (OM createOrUpdate subset). */
|
|
export interface OmTableUpsert {
|
|
name: string;
|
|
databaseSchema: string;
|
|
description?: string;
|
|
columns: { name: string; dataType: string; description?: string }[];
|
|
tags?: { tagFQN: string }[];
|
|
}
|
|
|
|
export interface MetadataProjectionResult {
|
|
projected: number;
|
|
failures: { dataset_id: string; error: string }[];
|
|
}
|
|
|
|
export class OpenMetadataAdapter {
|
|
private readonly baseUrl: string;
|
|
private readonly token?: string;
|
|
private readonly timeoutMs: number;
|
|
|
|
constructor(opts: OpenMetadataAdapterOptions) {
|
|
this.baseUrl = opts.baseUrl.replace(/\/+$/, "");
|
|
this.token = opts.token;
|
|
this.timeoutMs = opts.timeoutMs ?? 15_000;
|
|
}
|
|
|
|
async upsertTable(table: OmTableUpsert): Promise<void> {
|
|
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
|
if (this.token) headers["Authorization"] = `Bearer ${this.token}`;
|
|
let res: Response;
|
|
try {
|
|
res = await fetch(`${this.baseUrl}/api/v1/tables`, {
|
|
method: "PUT",
|
|
headers,
|
|
body: JSON.stringify(table),
|
|
signal: AbortSignal.timeout(this.timeoutMs),
|
|
});
|
|
} catch (cause) {
|
|
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "openmetadata", "PUT /api/v1/tables unreachable", { cause });
|
|
}
|
|
if (!res.ok) {
|
|
throw new SiaxError(
|
|
siaxErrorCodeForUpstreamStatus(res.status),
|
|
"openmetadata",
|
|
`PUT /api/v1/tables → HTTP ${res.status}`
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Projects 0pt's canonical dataset registry into OpenMetadata. Pure mapping +
|
|
* adapter calls; failures are collected per dataset, never silently dropped.
|
|
*/
|
|
export class MetadataProjectionService {
|
|
constructor(
|
|
private readonly adapter: OpenMetadataAdapter,
|
|
/** OM schema FQN the projection writes under, e.g. "0pt.default.datasets". */
|
|
private readonly schemaFqn: string = "0pt.default.datasets"
|
|
) {}
|
|
|
|
toTableUpsert(dataset: Dataset, latestVersion?: DatasetVersion): OmTableUpsert {
|
|
return {
|
|
name: dataset.dataset_id,
|
|
databaseSchema: this.schemaFqn,
|
|
description:
|
|
`${dataset.name} — PROJECTION of canonical 0pt dataset registry entry ` +
|
|
`0pt:dataset:${dataset.dataset_id}` +
|
|
(latestVersion ? ` (latest version ${latestVersion.version}, schema ${latestVersion.schema_version})` : "") +
|
|
`. Canonical truth lives in 0pt; this record is rebuildable.`,
|
|
columns: [
|
|
{ name: "dataset_id", dataType: "STRING", description: "Canonical 0pt dataset id" },
|
|
{ name: "tenant_id", dataType: "STRING", description: "Owning tenant" },
|
|
{ name: "dataset_type", dataType: "STRING", description: dataset.dataset_type },
|
|
{ name: "status", dataType: "STRING", description: dataset.status },
|
|
],
|
|
tags: [{ tagFQN: "0ptProjection.canonical-elsewhere" }],
|
|
};
|
|
}
|
|
|
|
async projectDataset(dataset: Dataset, latestVersion?: DatasetVersion): Promise<void> {
|
|
await this.adapter.upsertTable(this.toTableUpsert(dataset, latestVersion));
|
|
}
|
|
|
|
/** The §42 recovery path: re-project everything from canonical registries. */
|
|
async projectAll(
|
|
datasets: Dataset[],
|
|
latestVersions: Map<string, DatasetVersion>
|
|
): Promise<MetadataProjectionResult> {
|
|
const failures: { dataset_id: string; error: string }[] = [];
|
|
let projected = 0;
|
|
for (const ds of datasets) {
|
|
try {
|
|
await this.projectDataset(ds, latestVersions.get(ds.dataset_id));
|
|
projected++;
|
|
} catch (err) {
|
|
failures.push({ dataset_id: ds.dataset_id, error: err instanceof Error ? err.message : String(err) });
|
|
}
|
|
}
|
|
return { projected, failures };
|
|
}
|
|
}
|