feat(data-os): P0 capability layer — source registry, search, quality, dataset versions, metadata, analytics, features, ER

- Source Registry (§10): rights/license/purpose metadata, provenance,
  freshness SLA, health checks, describe/query; purpose gate FAILS CLOSED
  (no rights record => deny); cross-tenant probes => NOT_FOUND.
- Search (§10/§42): OpenSearch REST adapter (no SDK) + SearchService with
  MANDATORY tenant + purpose/rights filtering, versioned index definitions,
  bulk, generation-per-rebuild (opt-{name}--g{n}) with atomic alias cutover,
  rollback, failed-rebuild generation discard. OpenSearch never canonical.
- Quality (§10): versioned rule registry (GE-JSON interchange), local
  engine for the GE core subset (never silently passes unsupported types),
  runs/results/issues/score/trend, whole-run delegation to the GE backend
  via N0D (0pt.quality.ge_run) - never fabricated results.
- Dataset (§10/§5/§53): immutable numbered versions holding canonical
  st0re:artifact refs only, idempotent createVersion, export registers the
  ST0RE artifact + AUD0 evidence + OpenLineage.
- Metadata (§10): OpenMetadata PUT /api/v1/tables projection adapter +
  projectAll rebuild path; projection never canonical.
- Analytics (§10): ClickHouse HTTP adapter, closed metric-query DSL with
  FORCED tenant predicate + readonly/max_execution_time/max_result_rows,
  TTL retention. Zero connection to b0ts/quarantined pipelines.
- Features (§10): versioned feature-view registry, real Feast
  get-online-features adapter, materialize/get-historical as N0D delegation.
- ER (§10): generic candidate API with deterministic blocking + transparent
  scoring + per-field explanations, product hard-rule hook (force both
  directions), review-band handoff, Splink batch via N0D contract.

93 new unit/contract tests; suite 248 passed / 1 skipped.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Simon Axelsson
2026-08-31 16:49:32 +02:00
co-authored by Claude Fable 5
parent da58176735
commit b46807da75
19 changed files with 4038 additions and 0 deletions
+293
View File
@@ -0,0 +1,293 @@
/**
* src/services/dataset-version-service.ts
* ========================================
* P0 Dataset capability extensions (Master-DoD §10 Dataset:
* dataset.create/version/query/export) layered ON TOP of DatasetService (the
* owner of 0pt.dataset.schema.v1 registry entries — deliberately untouched:
* riv inte fungerande kod).
*
* dataset.version -> createVersion: an immutable, monotonically numbered
* version whose artifacts are CANONICAL ST0RE REFS
* (`st0re:artifact:art_x` — Master-DoD §5); 0pt never
* stores bytes here, only references.
* dataset.query -> listVersions/getVersion (tenant-gated).
* dataset.export -> exportDataset: registers the export artifact in ST0RE
* (real client), records AUD0 evidence + OpenLineage,
* returns the canonical `st0re:artifact:...` ref.
*
* Idempotency (§53): createVersion accepts an idempotency_key; the same key
* returns the SAME version instead of minting a duplicate.
*/
import crypto from "crypto";
import { SiaxError } from "../platform/errors";
import { formatResourceRef, isResourceRef } from "../platform/resource-ref";
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import { createSt0reClient, type St0reClient } from "../clients/st0re-client";
import { LineageEmitter } from "../lineage/lineage-emitter";
import { DatasetService } from "./dataset-service";
export interface DatasetVersion {
dataset_id: string;
version: number;
tenant_id: string;
project_id: string;
/** Canonical ST0RE artifact refs (st0re:artifact:...) making up this version. */
artifact_refs: string[];
/** Schema version of the dataset content at this version. */
schema_version: string;
/** Row/record count when known. */
record_count?: number;
description?: string;
created_at: string;
created_by: string;
correlation_id?: string;
}
export interface CreateDatasetVersionInput {
dataset_id: string;
tenant_id: string;
project_id: string;
artifact_refs: string[];
schema_version: string;
record_count?: number;
description?: string;
created_by: string;
correlation_id?: string;
idempotency_key?: string;
}
export interface DatasetExport {
export_id: string;
dataset_id: string;
version: number;
tenant_id: string;
/** Canonical ref of the registered export artifact. */
artifact_ref: string;
format: "parquet" | "csv" | "json";
created_at: string;
}
export class DatasetVersionService {
private readonly versions = new Map<string, DatasetVersion[]>();
private readonly idempotency = new Map<string, DatasetVersion>();
private readonly aud0: Aud0Client;
private readonly st0re: St0reClient;
private readonly lineage: LineageEmitter;
readonly datasets: DatasetService;
constructor(opts?: { aud0BaseUrl?: string; st0reBaseUrl?: string; datasetService?: DatasetService }) {
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.st0re = createSt0reClient(opts?.st0reBaseUrl);
this.lineage = new LineageEmitter(this.aud0);
this.datasets = opts?.datasetService ?? new DatasetService(opts?.aud0BaseUrl);
}
// -----------------------------------------------------------------------
// dataset.version
// -----------------------------------------------------------------------
async createVersion(input: CreateDatasetVersionInput): Promise<DatasetVersion> {
await this.assertDatasetTenant(input.dataset_id, input.tenant_id);
if (input.idempotency_key) {
const existing = this.idempotency.get(`${input.tenant_id}:${input.idempotency_key}`);
if (existing) return existing;
}
if (input.artifact_refs.length === 0) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "a dataset version needs at least one artifact ref");
}
for (const ref of input.artifact_refs) {
if (!isResourceRef(ref, "st0re", "artifact")) {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.dataset",
`artifact ref "${ref}" is not a canonical st0re:artifact:... reference (Master-DoD §5)`
);
}
}
const list = this.versions.get(input.dataset_id) ?? [];
const version: DatasetVersion = {
dataset_id: input.dataset_id,
version: (list[list.length - 1]?.version ?? 0) + 1,
tenant_id: input.tenant_id,
project_id: input.project_id,
artifact_refs: [...input.artifact_refs],
schema_version: input.schema_version,
record_count: input.record_count,
description: input.description,
created_at: new Date().toISOString(),
created_by: input.created_by,
correlation_id: input.correlation_id,
};
list.push(version);
this.versions.set(input.dataset_id, list);
if (input.idempotency_key) {
this.idempotency.set(`${input.tenant_id}:${input.idempotency_key}`, version);
}
const versionRef = formatResourceRef({
product: "0pt",
resourceType: "dataset",
resourceId: input.dataset_id,
version: String(version.version),
});
await this.aud0.ingestEvent({
event_id: `evt_0pt_dataset_version_${input.dataset_id}_v${version.version}`,
event_type: "0pt.dataset_version_created",
tenant_id: input.tenant_id,
project_id: input.project_id,
actor_id: input.created_by,
app_id: "0pt",
capability_id: "0pt.dataset.version.v1",
resource_type: "dataset_version",
resource_id: `${input.dataset_id}@${version.version}`,
risk_level: "L2",
result: "success",
input_refs: version.artifact_refs,
output_refs: [versionRef],
payload: { schema_version: input.schema_version, record_count: input.record_count },
created_at: version.created_at,
correlation_id: input.correlation_id,
idempotency_key: input.idempotency_key,
});
await this.lineage.emit({
tenant_id: input.tenant_id,
project_id: input.project_id,
capability_id: "0pt.dataset.version.v1",
run_id: `dataset-version:${input.dataset_id}:v${version.version}`,
event_type: "COMPLETE",
inputs: version.artifact_refs.map((ref) => ({ name: ref, ref })),
outputs: [{ name: `dataset/${input.dataset_id}`, ref: versionRef }],
correlation_id: input.correlation_id,
});
return version;
}
// -----------------------------------------------------------------------
// dataset.query
// -----------------------------------------------------------------------
async listVersions(datasetId: string, tenantId: string): Promise<DatasetVersion[]> {
await this.assertDatasetTenant(datasetId, tenantId);
return [...(this.versions.get(datasetId) ?? [])];
}
async getVersion(datasetId: string, tenantId: string, version: number): Promise<DatasetVersion> {
const list = await this.listVersions(datasetId, tenantId);
const found = list.find((v) => v.version === version);
if (!found) {
throw new SiaxError("NOT_FOUND", "0pt.dataset", `dataset ${datasetId} has no version ${version}`);
}
return found;
}
// -----------------------------------------------------------------------
// dataset.export
// -----------------------------------------------------------------------
async exportDataset(input: {
dataset_id: string;
tenant_id: string;
project_id: string;
version: number;
format: DatasetExport["format"];
/** Where the export bytes were written (by the exporting job). */
object_location: { provider: string; bucket: string; key: string; region?: string };
size_bytes: number;
checksum_sha256: string;
actor_id: string;
correlation_id?: string;
}): Promise<DatasetExport> {
const version = await this.getVersion(input.dataset_id, input.tenant_id, input.version);
const exportId = `dsexp_${crypto.randomUUID()}`;
const artifact = await this.st0re.createArtifact({
tenant_id: input.tenant_id,
project_id: input.project_id,
artifact_type: "dataset_export",
app_id: "0pt",
produced_by: "0pt.dataset.export.v1",
object_location: input.object_location,
content_type:
input.format === "parquet"
? "application/vnd.apache.parquet"
: input.format === "csv"
? "text/csv"
: "application/json",
size_bytes: input.size_bytes,
checksum_sha256: input.checksum_sha256,
pii_level: "none",
retention_policy: "standard",
data_residency: "eu",
risk_level: "L2",
correlation_id: input.correlation_id,
created_by: input.actor_id,
idempotency_key: `dsexp:${input.dataset_id}:v${input.version}:${input.format}:${input.checksum_sha256}`,
});
const artifactRef = formatResourceRef({
product: "st0re",
resourceType: "artifact",
resourceId: artifact.artifact_id,
});
const record: DatasetExport = {
export_id: exportId,
dataset_id: input.dataset_id,
version: input.version,
tenant_id: input.tenant_id,
artifact_ref: artifactRef,
format: input.format,
created_at: new Date().toISOString(),
};
const versionRef = formatResourceRef({
product: "0pt",
resourceType: "dataset",
resourceId: input.dataset_id,
version: String(version.version),
});
await this.aud0.ingestEvent({
event_id: `evt_0pt_dataset_export_${exportId}`,
event_type: "0pt.dataset_exported",
tenant_id: input.tenant_id,
project_id: input.project_id,
actor_id: input.actor_id,
app_id: "0pt",
capability_id: "0pt.dataset.export.v1",
resource_type: "dataset_export",
resource_id: exportId,
risk_level: "L3",
result: "success",
input_refs: [versionRef],
output_refs: [artifactRef],
payload: { format: input.format, size_bytes: input.size_bytes },
created_at: record.created_at,
correlation_id: input.correlation_id,
});
await this.lineage.emit({
tenant_id: input.tenant_id,
project_id: input.project_id,
capability_id: "0pt.dataset.export.v1",
run_id: `dataset-export:${exportId}`,
event_type: "COMPLETE",
inputs: [{ name: `dataset/${input.dataset_id}`, ref: versionRef }],
outputs: [{ name: artifactRef, ref: artifactRef }],
correlation_id: input.correlation_id,
});
return record;
}
// -----------------------------------------------------------------------
private async assertDatasetTenant(datasetId: string, tenantId: string): Promise<void> {
if (!tenantId) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "tenant_id is required");
let dataset;
try {
dataset = await this.datasets.getDataset(datasetId);
} catch {
throw new SiaxError("NOT_FOUND", "0pt.dataset", `dataset ${datasetId} not found`);
}
if (dataset.tenant_id !== tenantId) {
throw new SiaxError("NOT_FOUND", "0pt.dataset", `dataset ${datasetId} not found`);
}
}
}
+296
View File
@@ -0,0 +1,296 @@
/**
* src/services/source-registry-service.ts
* ========================================
* The P0 Source Registry capability surface (Master-DoD §10):
*
* source.register -> delegates to SourceService.createSource (owner of
* 0pt.source.schema.v1) + optional rights/provenance
* source.query -> listSources + rights/purpose filtering
* source.describe -> full SourceDescriptor (base + rights + provenance
* + health + freshness + rate policy)
* source.health -> record + aggregate health observations
* source.rate-policy -> already owned by SourceRatePolicyService; composed
*
* Tenant isolation: every read path REQUIRES a tenant_id and filters on it —
* a source registered by tenant A is invisible to tenant B (tested in
* tests/p0/source-registry.test.ts, incl. the cross-tenant attack case §50).
*
* Purpose enforcement: `assertPurposeAllowed` is the single hook other 0pt
* capabilities call before using a source for a given purpose (SearchService
* and DatasetService both do). No rights record ⇒ fail closed (deny).
*
* Persistence: in-memory Maps behind the same "pure core, thin registry"
* shape as the sibling P0 services. Production boot is protected by
* `resolveStoreMode` in the composition root (app/api routes construct with
* the shared registry): with no database configured a production runtime
* throws DEPENDENCY_UNAVAILABLE instead of serving Maps (Master-DoD §3).
*/
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import { SiaxError } from "../platform/errors";
import { SourceService } from "./source-service";
import { SourceRatePolicyService } from "./source-rate-policy-service";
import type { CreateSourceInput, Source } from "../types/source";
import type {
SourceDescriptor,
SourceFreshness,
SourceHealth,
SourceHealthCheck,
SourceHealthStatus,
SourceProvenance,
SourceRights,
} from "../types/source-registry";
export interface RegisterSourceInput extends CreateSourceInput {
rights?: Omit<SourceRights, "source_id" | "tenant_id">;
provenance?: Omit<SourceProvenance, "source_id" | "tenant_id" | "updated_at">;
}
export class SourceRegistryService {
private readonly rights = new Map<string, SourceRights>();
private readonly provenance = new Map<string, SourceProvenance>();
private readonly healthChecks = new Map<string, SourceHealthCheck[]>();
private readonly lastSuccess = new Map<string, string>();
private readonly aud0: Aud0Client;
readonly sources: SourceService;
readonly ratePolicies: SourceRatePolicyService;
constructor(opts?: {
aud0BaseUrl?: string;
sourceService?: SourceService;
ratePolicyService?: SourceRatePolicyService;
}) {
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.sources = opts?.sourceService ?? new SourceService(opts?.aud0BaseUrl);
this.ratePolicies =
opts?.ratePolicyService ??
new SourceRatePolicyService({ aud0BaseUrl: opts?.aud0BaseUrl, sourceService: this.sources });
}
// -------------------------------------------------------------------------
// source.register
// -------------------------------------------------------------------------
async registerSource(input: RegisterSourceInput): Promise<SourceDescriptor> {
const { rights, provenance, ...base } = input;
const source = await this.sources.createSource(base);
if (rights) {
await this.setRights({ ...rights, source_id: source.source_id, tenant_id: source.tenant_id });
}
if (provenance) {
await this.setProvenance({
...provenance,
source_id: source.source_id,
tenant_id: source.tenant_id,
updated_at: new Date().toISOString(),
});
}
return this.describeSource(source.source_id, source.tenant_id);
}
// -------------------------------------------------------------------------
// rights / license / purpose metadata
// -------------------------------------------------------------------------
async setRights(rights: SourceRights): Promise<SourceRights> {
await this.assertSourceTenant(rights.source_id, rights.tenant_id);
this.rights.set(rights.source_id, rights);
await this.emit("0pt.source_rights_set", rights.tenant_id, rights.source_id, {
license_id: rights.license_id,
allowed_purposes: rights.allowed_purposes,
redistribution: rights.redistribution,
});
return rights;
}
async getRights(sourceId: string, tenantId: string): Promise<SourceRights | undefined> {
await this.assertSourceTenant(sourceId, tenantId);
return this.rights.get(sourceId);
}
/**
* The purpose gate other capabilities call before using a source. FAIL
* CLOSED: no rights record, or a purpose outside the allowlist, throws
* AUTHORIZATION_DENIED.
*/
async assertPurposeAllowed(sourceId: string, tenantId: string, purpose: string): Promise<SourceRights> {
await this.assertSourceTenant(sourceId, tenantId);
const rights = this.rights.get(sourceId);
if (!rights) {
throw new SiaxError(
"AUTHORIZATION_DENIED",
"0pt.source_registry",
`source ${sourceId} has no rights record — purpose "${purpose}" denied (fail closed)`
);
}
if (!rights.allowed_purposes.includes(purpose)) {
throw new SiaxError(
"AUTHORIZATION_DENIED",
"0pt.source_registry",
`purpose "${purpose}" is not in source ${sourceId}'s allowed purposes`
);
}
return rights;
}
// -------------------------------------------------------------------------
// provenance / freshness
// -------------------------------------------------------------------------
async setProvenance(prov: SourceProvenance): Promise<SourceProvenance> {
await this.assertSourceTenant(prov.source_id, prov.tenant_id);
this.provenance.set(prov.source_id, prov);
await this.emit("0pt.source_provenance_set", prov.tenant_id, prov.source_id, {
method: prov.method,
authority: prov.authority,
cadence: prov.cadence,
});
return prov;
}
async getProvenance(sourceId: string, tenantId: string): Promise<SourceProvenance | undefined> {
await this.assertSourceTenant(sourceId, tenantId);
return this.provenance.get(sourceId);
}
/** Record a successful data observation (drives freshness). */
async recordSuccess(sourceId: string, tenantId: string, atIso?: string): Promise<void> {
await this.assertSourceTenant(sourceId, tenantId);
this.lastSuccess.set(sourceId, atIso ?? new Date().toISOString());
}
getFreshness(sourceId: string, now: Date = new Date()): SourceFreshness {
const prov = this.provenance.get(sourceId);
const last = this.lastSuccess.get(sourceId);
const sla = prov?.freshness_sla_hours;
if (!last) {
return { source_id: sourceId, freshness_sla_hours: sla, fresh: sla === undefined };
}
const ageHours = (now.getTime() - new Date(last).getTime()) / 3_600_000;
return {
source_id: sourceId,
last_success_at: last,
freshness_sla_hours: sla,
age_hours: Math.round(ageHours * 100) / 100,
fresh: sla === undefined ? true : ageHours <= sla,
};
}
// -------------------------------------------------------------------------
// source.health
// -------------------------------------------------------------------------
async recordHealthCheck(check: SourceHealthCheck): Promise<SourceHealth> {
await this.assertSourceTenant(check.source_id, check.tenant_id);
const list = this.healthChecks.get(check.source_id) ?? [];
list.push(check);
// Bounded history — health is an operational signal, not an audit log
// (AUD0 owns history); keep the latest 100 observations.
if (list.length > 100) list.splice(0, list.length - 100);
this.healthChecks.set(check.source_id, list);
if (check.status === "up") {
await this.recordSuccess(check.source_id, check.tenant_id, check.checked_at);
}
return this.getHealth(check.source_id);
}
getHealth(sourceId: string): SourceHealth {
const list = this.healthChecks.get(sourceId) ?? [];
const last = list[list.length - 1];
let consecutiveFailures = 0;
for (let i = list.length - 1; i >= 0; i--) {
if (list[i].status === "down" || list[i].status === "degraded") consecutiveFailures++;
else break;
}
const status: SourceHealthStatus = last?.status ?? "unknown";
return { source_id: sourceId, status, last_check: last, consecutive_failures: consecutiveFailures };
}
// -------------------------------------------------------------------------
// source.query / source.describe
// -------------------------------------------------------------------------
async querySources(filter: {
tenant_id: string;
project_id?: string;
source_type?: Source["source_type"];
status?: Source["status"];
/** Only sources whose rights allowlist includes this purpose. */
purpose?: string;
/** Only sources with at most this redistribution openness. */
redistribution?: SourceRights["redistribution"];
}): Promise<Source[]> {
if (!filter.tenant_id) {
throw new SiaxError("INVALID_REQUEST", "0pt.source_registry", "tenant_id is required on source.query");
}
let results = await this.sources.listSources({
tenant_id: filter.tenant_id,
project_id: filter.project_id,
source_type: filter.source_type,
status: filter.status,
});
if (filter.purpose !== undefined) {
results = results.filter((s) => {
const rights = this.rights.get(s.source_id);
return rights !== undefined && rights.allowed_purposes.includes(filter.purpose as string);
});
}
if (filter.redistribution !== undefined) {
results = results.filter((s) => this.rights.get(s.source_id)?.redistribution === filter.redistribution);
}
return results;
}
async describeSource(sourceId: string, tenantId: string): Promise<SourceDescriptor> {
const source = await this.assertSourceTenant(sourceId, tenantId);
const policies = await this.ratePolicies.listSourceRatePolicies({ tenant_id: tenantId, source_id: sourceId });
return {
source,
rights: this.rights.get(sourceId),
provenance: this.provenance.get(sourceId),
health: this.getHealth(sourceId),
freshness: this.getFreshness(sourceId),
rate_policy: policies[0],
};
}
// -------------------------------------------------------------------------
// internals
// -------------------------------------------------------------------------
/**
* Tenant gate for every read/write: resolves the source AND verifies the
* caller's tenant owns it. Cross-tenant probes get NOT_FOUND (no existence
* oracle), same 404-not-403 posture as ST0RE/N0D.
*/
private async assertSourceTenant(sourceId: string, tenantId: string): Promise<Source> {
if (!tenantId) {
throw new SiaxError("INVALID_REQUEST", "0pt.source_registry", "tenant_id is required");
}
let source: Source;
try {
source = await this.sources.getSource(sourceId);
} catch {
throw new SiaxError("NOT_FOUND", "0pt.source_registry", `source ${sourceId} not found`);
}
if (source.tenant_id !== tenantId) {
throw new SiaxError("NOT_FOUND", "0pt.source_registry", `source ${sourceId} not found`);
}
return source;
}
private async emit(
eventType: string,
tenantId: string,
sourceId: string,
payload: Record<string, unknown>
): Promise<void> {
await this.aud0.ingestEvent({
event_id: `evt_${eventType}_${sourceId}_${Date.now()}`,
event_type: eventType,
tenant_id: tenantId,
project_id: "p_platform",
actor_id: "svc_0pt",
app_id: "0pt",
capability_id: eventType.replace(/^0pt\./, "0pt."),
resource_type: "source",
resource_id: sourceId,
risk_level: "L2",
payload,
created_at: new Date().toISOString(),
});
}
}