feat(data-os): P0 capability layer — source registry, search, quality, dataset versions, metadata, analytics, features, ER

- Source Registry (§10): rights/license/purpose metadata, provenance,
  freshness SLA, health checks, describe/query; purpose gate FAILS CLOSED
  (no rights record => deny); cross-tenant probes => NOT_FOUND.
- Search (§10/§42): OpenSearch REST adapter (no SDK) + SearchService with
  MANDATORY tenant + purpose/rights filtering, versioned index definitions,
  bulk, generation-per-rebuild (opt-{name}--g{n}) with atomic alias cutover,
  rollback, failed-rebuild generation discard. OpenSearch never canonical.
- Quality (§10): versioned rule registry (GE-JSON interchange), local
  engine for the GE core subset (never silently passes unsupported types),
  runs/results/issues/score/trend, whole-run delegation to the GE backend
  via N0D (0pt.quality.ge_run) - never fabricated results.
- Dataset (§10/§5/§53): immutable numbered versions holding canonical
  st0re:artifact refs only, idempotent createVersion, export registers the
  ST0RE artifact + AUD0 evidence + OpenLineage.
- Metadata (§10): OpenMetadata PUT /api/v1/tables projection adapter +
  projectAll rebuild path; projection never canonical.
- Analytics (§10): ClickHouse HTTP adapter, closed metric-query DSL with
  FORCED tenant predicate + readonly/max_execution_time/max_result_rows,
  TTL retention. Zero connection to b0ts/quarantined pipelines.
- Features (§10): versioned feature-view registry, real Feast
  get-online-features adapter, materialize/get-historical as N0D delegation.
- ER (§10): generic candidate API with deterministic blocking + transparent
  scoring + per-field explanations, product hard-rule hook (force both
  directions), review-band handoff, Splink batch via N0D contract.

93 new unit/contract tests; suite 248 passed / 1 skipped.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Simon Axelsson
2026-08-31 16:49:32 +02:00
co-authored by Claude Fable 5
parent da58176735
commit b46807da75
19 changed files with 4038 additions and 0 deletions
+98
View File
@@ -0,0 +1,98 @@
/**
* src/types/source-registry.ts
* =============================
* P0 Source Registry extensions (Master-DoD §10 Source Registry: rights,
* license, purpose metadata, health, freshness, provenance) layered ON TOP of
* the existing 0pt.source.schema.v1 (src/types/source.ts). Additive — the
* base Source type is unchanged; these records reference `source_id`.
*
* Vocabulary notes:
* - `allowed_purposes` uses the same GDPR/purpose-limitation tags as the
* SIAX request context (`purpose`), so a query path can enforce
* purpose-fit mechanically (SearchService does exactly that).
* - `access` mirrors the legal-sourcing vocabulary from
* docs/03-data-model.md / CLAUDE.md §1 (api | bulk | authorized_web |
* licensed) — 0pt never has a "bypass" access mode.
*/
/** Rights/license/purpose metadata for one registered source (§10). */
export interface SourceRights {
source_id: string;
tenant_id: string;
/** SPDX id or license registry key, e.g. "CC-BY-4.0", "proprietary:carinfo-2026". */
license_id: string;
license_url?: string;
/** Purposes this source's data may be used for. Empty = no purpose approved (fail closed). */
allowed_purposes: string[];
/** Whether records from this source may contain personal data at all. */
pii_allowed: boolean;
/** How far results derived from this source may travel. */
redistribution: "internal" | "customer_deliverable" | "public" | "prohibited";
/** Attribution string that MUST accompany redistributed data, when required. */
attribution?: string;
/** ISO date the rights assessment was made/verified. */
verified_at: string;
/** Who made the assessment (actor id). */
verified_by: string;
notes?: string;
}
/** Provenance/method metadata: HOW data is obtained from the source (§10). */
export interface SourceProvenance {
source_id: string;
tenant_id: string;
/** Legal sourcing method (CLAUDE.md §1 vocabulary; no bypass modes exist). */
method: "official_api" | "bulk_download" | "authorized_web" | "licensed_feed" | "manual_upload";
/** Upstream authority, e.g. "Bolagsverket", "RDW". */
authority?: string;
/** Robots/ToS evidence ref (1p evidence or aud0 evidence ref) where applicable. */
compliance_evidence_ref?: string;
/** Expected cadence of upstream updates, e.g. "daily", "weekly", "on_change". */
cadence?: string;
/** Freshness SLA: data older than this many hours counts as stale. */
freshness_sla_hours?: number;
updated_at: string;
}
export type SourceHealthStatus = "up" | "degraded" | "down" | "unknown";
/** One recorded health observation for a source. */
export interface SourceHealthCheck {
source_id: string;
tenant_id: string;
status: SourceHealthStatus;
checked_at: string;
latency_ms?: number;
/** Short operator-facing detail; never raw upstream bodies. */
detail?: string;
}
/** Aggregate health view returned by source.health. */
export interface SourceHealth {
source_id: string;
status: SourceHealthStatus;
last_check?: SourceHealthCheck;
/** Consecutive failures as of the last check. */
consecutive_failures: number;
}
/** Freshness view derived from provenance SLA + last successful observation. */
export interface SourceFreshness {
source_id: string;
/** Last time data from this source was successfully obtained/observed. */
last_success_at?: string;
freshness_sla_hours?: number;
/** true when last_success_at is within the SLA (or no SLA is declared). */
fresh: boolean;
age_hours?: number;
}
/** The full `source.describe` document (§10): base + rights + provenance + health + freshness + rate policy. */
export interface SourceDescriptor {
source: import("./source").Source;
rights?: SourceRights;
provenance?: SourceProvenance;
health: SourceHealth;
freshness: SourceFreshness;
rate_policy?: import("./source-rate-policy").SourceRatePolicy;
}