- Source Registry (§10): rights/license/purpose metadata, provenance,
freshness SLA, health checks, describe/query; purpose gate FAILS CLOSED
(no rights record => deny); cross-tenant probes => NOT_FOUND.
- Search (§10/§42): OpenSearch REST adapter (no SDK) + SearchService with
MANDATORY tenant + purpose/rights filtering, versioned index definitions,
bulk, generation-per-rebuild (opt-{name}--g{n}) with atomic alias cutover,
rollback, failed-rebuild generation discard. OpenSearch never canonical.
- Quality (§10): versioned rule registry (GE-JSON interchange), local
engine for the GE core subset (never silently passes unsupported types),
runs/results/issues/score/trend, whole-run delegation to the GE backend
via N0D (0pt.quality.ge_run) - never fabricated results.
- Dataset (§10/§5/§53): immutable numbered versions holding canonical
st0re:artifact refs only, idempotent createVersion, export registers the
ST0RE artifact + AUD0 evidence + OpenLineage.
- Metadata (§10): OpenMetadata PUT /api/v1/tables projection adapter +
projectAll rebuild path; projection never canonical.
- Analytics (§10): ClickHouse HTTP adapter, closed metric-query DSL with
FORCED tenant predicate + readonly/max_execution_time/max_result_rows,
TTL retention. Zero connection to b0ts/quarantined pipelines.
- Features (§10): versioned feature-view registry, real Feast
get-online-features adapter, materialize/get-historical as N0D delegation.
- ER (§10): generic candidate API with deterministic blocking + transparent
scoring + per-field explanations, product hard-rule hook (force both
directions), review-band handoff, Splink batch via N0D contract.
93 new unit/contract tests; suite 248 passed / 1 skipped.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
297 lines
12 KiB
TypeScript
297 lines
12 KiB
TypeScript
/**
|
|
* src/services/source-registry-service.ts
|
|
* ========================================
|
|
* The P0 Source Registry capability surface (Master-DoD §10):
|
|
*
|
|
* source.register -> delegates to SourceService.createSource (owner of
|
|
* 0pt.source.schema.v1) + optional rights/provenance
|
|
* source.query -> listSources + rights/purpose filtering
|
|
* source.describe -> full SourceDescriptor (base + rights + provenance
|
|
* + health + freshness + rate policy)
|
|
* source.health -> record + aggregate health observations
|
|
* source.rate-policy -> already owned by SourceRatePolicyService; composed
|
|
*
|
|
* Tenant isolation: every read path REQUIRES a tenant_id and filters on it —
|
|
* a source registered by tenant A is invisible to tenant B (tested in
|
|
* tests/p0/source-registry.test.ts, incl. the cross-tenant attack case §50).
|
|
*
|
|
* Purpose enforcement: `assertPurposeAllowed` is the single hook other 0pt
|
|
* capabilities call before using a source for a given purpose (SearchService
|
|
* and DatasetService both do). No rights record ⇒ fail closed (deny).
|
|
*
|
|
* Persistence: in-memory Maps behind the same "pure core, thin registry"
|
|
* shape as the sibling P0 services. Production boot is protected by
|
|
* `resolveStoreMode` in the composition root (app/api routes construct with
|
|
* the shared registry): with no database configured a production runtime
|
|
* throws DEPENDENCY_UNAVAILABLE instead of serving Maps (Master-DoD §3).
|
|
*/
|
|
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
|
import { SiaxError } from "../platform/errors";
|
|
import { SourceService } from "./source-service";
|
|
import { SourceRatePolicyService } from "./source-rate-policy-service";
|
|
import type { CreateSourceInput, Source } from "../types/source";
|
|
import type {
|
|
SourceDescriptor,
|
|
SourceFreshness,
|
|
SourceHealth,
|
|
SourceHealthCheck,
|
|
SourceHealthStatus,
|
|
SourceProvenance,
|
|
SourceRights,
|
|
} from "../types/source-registry";
|
|
|
|
export interface RegisterSourceInput extends CreateSourceInput {
|
|
rights?: Omit<SourceRights, "source_id" | "tenant_id">;
|
|
provenance?: Omit<SourceProvenance, "source_id" | "tenant_id" | "updated_at">;
|
|
}
|
|
|
|
export class SourceRegistryService {
|
|
private readonly rights = new Map<string, SourceRights>();
|
|
private readonly provenance = new Map<string, SourceProvenance>();
|
|
private readonly healthChecks = new Map<string, SourceHealthCheck[]>();
|
|
private readonly lastSuccess = new Map<string, string>();
|
|
private readonly aud0: Aud0Client;
|
|
readonly sources: SourceService;
|
|
readonly ratePolicies: SourceRatePolicyService;
|
|
|
|
constructor(opts?: {
|
|
aud0BaseUrl?: string;
|
|
sourceService?: SourceService;
|
|
ratePolicyService?: SourceRatePolicyService;
|
|
}) {
|
|
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
|
this.sources = opts?.sourceService ?? new SourceService(opts?.aud0BaseUrl);
|
|
this.ratePolicies =
|
|
opts?.ratePolicyService ??
|
|
new SourceRatePolicyService({ aud0BaseUrl: opts?.aud0BaseUrl, sourceService: this.sources });
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// source.register
|
|
// -------------------------------------------------------------------------
|
|
async registerSource(input: RegisterSourceInput): Promise<SourceDescriptor> {
|
|
const { rights, provenance, ...base } = input;
|
|
const source = await this.sources.createSource(base);
|
|
if (rights) {
|
|
await this.setRights({ ...rights, source_id: source.source_id, tenant_id: source.tenant_id });
|
|
}
|
|
if (provenance) {
|
|
await this.setProvenance({
|
|
...provenance,
|
|
source_id: source.source_id,
|
|
tenant_id: source.tenant_id,
|
|
updated_at: new Date().toISOString(),
|
|
});
|
|
}
|
|
return this.describeSource(source.source_id, source.tenant_id);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// rights / license / purpose metadata
|
|
// -------------------------------------------------------------------------
|
|
async setRights(rights: SourceRights): Promise<SourceRights> {
|
|
await this.assertSourceTenant(rights.source_id, rights.tenant_id);
|
|
this.rights.set(rights.source_id, rights);
|
|
await this.emit("0pt.source_rights_set", rights.tenant_id, rights.source_id, {
|
|
license_id: rights.license_id,
|
|
allowed_purposes: rights.allowed_purposes,
|
|
redistribution: rights.redistribution,
|
|
});
|
|
return rights;
|
|
}
|
|
|
|
async getRights(sourceId: string, tenantId: string): Promise<SourceRights | undefined> {
|
|
await this.assertSourceTenant(sourceId, tenantId);
|
|
return this.rights.get(sourceId);
|
|
}
|
|
|
|
/**
|
|
* The purpose gate other capabilities call before using a source. FAIL
|
|
* CLOSED: no rights record, or a purpose outside the allowlist, throws
|
|
* AUTHORIZATION_DENIED.
|
|
*/
|
|
async assertPurposeAllowed(sourceId: string, tenantId: string, purpose: string): Promise<SourceRights> {
|
|
await this.assertSourceTenant(sourceId, tenantId);
|
|
const rights = this.rights.get(sourceId);
|
|
if (!rights) {
|
|
throw new SiaxError(
|
|
"AUTHORIZATION_DENIED",
|
|
"0pt.source_registry",
|
|
`source ${sourceId} has no rights record — purpose "${purpose}" denied (fail closed)`
|
|
);
|
|
}
|
|
if (!rights.allowed_purposes.includes(purpose)) {
|
|
throw new SiaxError(
|
|
"AUTHORIZATION_DENIED",
|
|
"0pt.source_registry",
|
|
`purpose "${purpose}" is not in source ${sourceId}'s allowed purposes`
|
|
);
|
|
}
|
|
return rights;
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// provenance / freshness
|
|
// -------------------------------------------------------------------------
|
|
async setProvenance(prov: SourceProvenance): Promise<SourceProvenance> {
|
|
await this.assertSourceTenant(prov.source_id, prov.tenant_id);
|
|
this.provenance.set(prov.source_id, prov);
|
|
await this.emit("0pt.source_provenance_set", prov.tenant_id, prov.source_id, {
|
|
method: prov.method,
|
|
authority: prov.authority,
|
|
cadence: prov.cadence,
|
|
});
|
|
return prov;
|
|
}
|
|
|
|
async getProvenance(sourceId: string, tenantId: string): Promise<SourceProvenance | undefined> {
|
|
await this.assertSourceTenant(sourceId, tenantId);
|
|
return this.provenance.get(sourceId);
|
|
}
|
|
|
|
/** Record a successful data observation (drives freshness). */
|
|
async recordSuccess(sourceId: string, tenantId: string, atIso?: string): Promise<void> {
|
|
await this.assertSourceTenant(sourceId, tenantId);
|
|
this.lastSuccess.set(sourceId, atIso ?? new Date().toISOString());
|
|
}
|
|
|
|
getFreshness(sourceId: string, now: Date = new Date()): SourceFreshness {
|
|
const prov = this.provenance.get(sourceId);
|
|
const last = this.lastSuccess.get(sourceId);
|
|
const sla = prov?.freshness_sla_hours;
|
|
if (!last) {
|
|
return { source_id: sourceId, freshness_sla_hours: sla, fresh: sla === undefined };
|
|
}
|
|
const ageHours = (now.getTime() - new Date(last).getTime()) / 3_600_000;
|
|
return {
|
|
source_id: sourceId,
|
|
last_success_at: last,
|
|
freshness_sla_hours: sla,
|
|
age_hours: Math.round(ageHours * 100) / 100,
|
|
fresh: sla === undefined ? true : ageHours <= sla,
|
|
};
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// source.health
|
|
// -------------------------------------------------------------------------
|
|
async recordHealthCheck(check: SourceHealthCheck): Promise<SourceHealth> {
|
|
await this.assertSourceTenant(check.source_id, check.tenant_id);
|
|
const list = this.healthChecks.get(check.source_id) ?? [];
|
|
list.push(check);
|
|
// Bounded history — health is an operational signal, not an audit log
|
|
// (AUD0 owns history); keep the latest 100 observations.
|
|
if (list.length > 100) list.splice(0, list.length - 100);
|
|
this.healthChecks.set(check.source_id, list);
|
|
if (check.status === "up") {
|
|
await this.recordSuccess(check.source_id, check.tenant_id, check.checked_at);
|
|
}
|
|
return this.getHealth(check.source_id);
|
|
}
|
|
|
|
getHealth(sourceId: string): SourceHealth {
|
|
const list = this.healthChecks.get(sourceId) ?? [];
|
|
const last = list[list.length - 1];
|
|
let consecutiveFailures = 0;
|
|
for (let i = list.length - 1; i >= 0; i--) {
|
|
if (list[i].status === "down" || list[i].status === "degraded") consecutiveFailures++;
|
|
else break;
|
|
}
|
|
const status: SourceHealthStatus = last?.status ?? "unknown";
|
|
return { source_id: sourceId, status, last_check: last, consecutive_failures: consecutiveFailures };
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// source.query / source.describe
|
|
// -------------------------------------------------------------------------
|
|
async querySources(filter: {
|
|
tenant_id: string;
|
|
project_id?: string;
|
|
source_type?: Source["source_type"];
|
|
status?: Source["status"];
|
|
/** Only sources whose rights allowlist includes this purpose. */
|
|
purpose?: string;
|
|
/** Only sources with at most this redistribution openness. */
|
|
redistribution?: SourceRights["redistribution"];
|
|
}): Promise<Source[]> {
|
|
if (!filter.tenant_id) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.source_registry", "tenant_id is required on source.query");
|
|
}
|
|
let results = await this.sources.listSources({
|
|
tenant_id: filter.tenant_id,
|
|
project_id: filter.project_id,
|
|
source_type: filter.source_type,
|
|
status: filter.status,
|
|
});
|
|
if (filter.purpose !== undefined) {
|
|
results = results.filter((s) => {
|
|
const rights = this.rights.get(s.source_id);
|
|
return rights !== undefined && rights.allowed_purposes.includes(filter.purpose as string);
|
|
});
|
|
}
|
|
if (filter.redistribution !== undefined) {
|
|
results = results.filter((s) => this.rights.get(s.source_id)?.redistribution === filter.redistribution);
|
|
}
|
|
return results;
|
|
}
|
|
|
|
async describeSource(sourceId: string, tenantId: string): Promise<SourceDescriptor> {
|
|
const source = await this.assertSourceTenant(sourceId, tenantId);
|
|
const policies = await this.ratePolicies.listSourceRatePolicies({ tenant_id: tenantId, source_id: sourceId });
|
|
return {
|
|
source,
|
|
rights: this.rights.get(sourceId),
|
|
provenance: this.provenance.get(sourceId),
|
|
health: this.getHealth(sourceId),
|
|
freshness: this.getFreshness(sourceId),
|
|
rate_policy: policies[0],
|
|
};
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// internals
|
|
// -------------------------------------------------------------------------
|
|
/**
|
|
* Tenant gate for every read/write: resolves the source AND verifies the
|
|
* caller's tenant owns it. Cross-tenant probes get NOT_FOUND (no existence
|
|
* oracle), same 404-not-403 posture as ST0RE/N0D.
|
|
*/
|
|
private async assertSourceTenant(sourceId: string, tenantId: string): Promise<Source> {
|
|
if (!tenantId) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.source_registry", "tenant_id is required");
|
|
}
|
|
let source: Source;
|
|
try {
|
|
source = await this.sources.getSource(sourceId);
|
|
} catch {
|
|
throw new SiaxError("NOT_FOUND", "0pt.source_registry", `source ${sourceId} not found`);
|
|
}
|
|
if (source.tenant_id !== tenantId) {
|
|
throw new SiaxError("NOT_FOUND", "0pt.source_registry", `source ${sourceId} not found`);
|
|
}
|
|
return source;
|
|
}
|
|
|
|
private async emit(
|
|
eventType: string,
|
|
tenantId: string,
|
|
sourceId: string,
|
|
payload: Record<string, unknown>
|
|
): Promise<void> {
|
|
await this.aud0.ingestEvent({
|
|
event_id: `evt_${eventType}_${sourceId}_${Date.now()}`,
|
|
event_type: eventType,
|
|
tenant_id: tenantId,
|
|
project_id: "p_platform",
|
|
actor_id: "svc_0pt",
|
|
app_id: "0pt",
|
|
capability_id: eventType.replace(/^0pt\./, "0pt."),
|
|
resource_type: "source",
|
|
resource_id: sourceId,
|
|
risk_level: "L2",
|
|
payload,
|
|
created_at: new Date().toISOString(),
|
|
});
|
|
}
|
|
}
|