Files
0pt/src/services/source-registry-service.ts
T
Simon AxelssonandClaude Sonnet 5 07b13bd833 feat(data-os): Source Registry Postgres persistence + full HTTP route surface
Replaces the in-memory (per-instance, not even per-process-shared) Maps in
SourceService/SourceRegistryService with real Postgres persistence, and adds
the register/query/describe/health/rights/provenance/freshness HTTP routes
that never existed (grep for SourceRegistryService in app/ previously
returned nothing).

- sql/002_source_registry.sql: opt_sources, opt_source_rights,
  opt_source_provenance, opt_source_health_checks, opt_source_freshness.
  Numbered 002 (not 006) because 004's opt_source_rate_policies already
  forward-references opt_sources, which no sql/ file defined until now —
  fixes 004's pre-existing broken migration ordering (opt_datasets remains
  a separate, unrelated, still-open gap in 004, flagged not fixed).
- src/repositories/*: SourceRepository / SourceRegistryRepository seams,
  each with an in-memory implementation (business logic extracted verbatim)
  and a Postgres implementation, chosen via resolveStoreMode — production
  still fails hard with no DATABASE_URL (Master-DoD §3), but a configured
  production now actually works instead of always being blocked.
- SourceService / SourceRegistryService: storage moved behind the
  repository seam; getHealth/getFreshness are now async and tenant-scoped
  at the accessor itself (needed once they're reachable directly over HTTP,
  not only after an earlier caller's tenant check).
- src/services/source-registry.ts: shared singleton instances for the
  sources/* route group, fixing the "each route module owned its own
  private registry" defect (source-rate-policies and .../query previously
  couldn't see a source registered via POST /api/0pt/sources).
- New routes: POST/GET /api/0pt/sources, GET /api/0pt/sources/[id],
  GET+PUT .../rights, GET+PUT .../provenance, GET+POST .../health,
  GET .../freshness.
- Business logic unchanged: fail-closed purpose gate, cross-tenant
  NOT_FOUND (no existence oracle, §50), bounded 100-row health history.

Tested against a real Postgres container (not mocked) —
tests/p0/source-registry-postgres.realinfra.test.ts, env-gated on
SOURCE_REGISTRY_TEST_DATABASE_URL, skips honestly when unset. All 249
pre-existing P0 tests + 7 new realinfra tests pass; also smoke-tested the
full HTTP route surface end-to-end against next dev + live Postgres.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-01 20:42:41 +02:00

328 lines
13 KiB
TypeScript

/**
* src/services/source-registry-service.ts
* ========================================
* The P0 Source Registry capability surface (Master-DoD §10):
*
* source.register -> delegates to SourceService.createSource (owner of
* 0pt.source.schema.v1) + optional rights/provenance
* source.query -> listSources + rights/purpose filtering
* source.describe -> full SourceDescriptor (base + rights + provenance
* + health + freshness + rate policy)
* source.health -> record + aggregate health observations
* source.rate-policy -> already owned by SourceRatePolicyService; composed
*
* Tenant isolation: every read path REQUIRES a tenant_id and filters on it —
* a source registered by tenant A is invisible to tenant B (tested in
* tests/p0/source-registry.test.ts, incl. the cross-tenant attack case §50).
* `getHealth`/`getFreshness` now take tenant_id directly and run the same
* `assertSourceTenant` gate every other accessor does — they are exposed as
* standalone HTTP routes now (app/api/0pt/sources/[sourceId]/health,
* .../freshness), so they can no longer rely on an earlier caller having
* already checked tenancy on their behalf.
*
* Purpose enforcement: `assertPurposeAllowed` is the single hook other 0pt
* capabilities call before using a source for a given purpose (SearchService
* and DatasetService both do). No rights record ⇒ fail closed (deny).
*
* Persistence: rights/provenance/health/freshness now live behind
* SourceRegistryRepository (src/repositories/source-registry-repository.ts)
* — real Postgres (`opt_source_rights`/`opt_source_provenance`/
* `opt_source_health_checks`/`opt_source_freshness`,
* sql/002_source_registry.sql) when DATABASE_URL is configured, in-memory
* Maps otherwise. `resolveStoreMode` still fails production boot hard with
* no DATABASE_URL (Master-DoD §3) — the difference from before is that a
* configured production now actually WORKS (opt_sources/etc. exist), not
* just fails safely.
*/
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import { SiaxError } from "../platform/errors";
import {
createSourceRegistryRepository,
type SourceRegistryRepository,
} from "../repositories/source-registry-repository";
import { SourceService, type SourceServiceOptions } from "./source-service";
import { SourceRatePolicyService } from "./source-rate-policy-service";
import type { CreateSourceInput, Source } from "../types/source";
import type {
SourceDescriptor,
SourceFreshness,
SourceHealth,
SourceHealthCheck,
SourceHealthStatus,
SourceProvenance,
SourceRights,
} from "../types/source-registry";
export interface RegisterSourceInput extends CreateSourceInput {
rights?: Omit<SourceRights, "source_id" | "tenant_id">;
provenance?: Omit<SourceProvenance, "source_id" | "tenant_id" | "updated_at">;
}
export class SourceRegistryService {
private readonly repo: SourceRegistryRepository;
private readonly aud0: Aud0Client;
readonly sources: SourceService;
readonly ratePolicies: SourceRatePolicyService;
constructor(opts?: {
aud0BaseUrl?: string;
/** Explicit DATABASE_URL override — mainly for tests. Falls back to resolveStoreMode's own env lookup when omitted. */
databaseUrl?: string;
sourceService?: SourceService;
ratePolicyService?: SourceRatePolicyService;
/** Inject a repository directly (tests) — bypasses resolveStoreMode entirely. */
repository?: SourceRegistryRepository;
}) {
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.repo = opts?.repository ?? createSourceRegistryRepository(opts?.databaseUrl);
const sourceServiceOpts: SourceServiceOptions = {
aud0BaseUrl: opts?.aud0BaseUrl,
databaseUrl: opts?.databaseUrl,
};
this.sources = opts?.sourceService ?? new SourceService(sourceServiceOpts);
this.ratePolicies =
opts?.ratePolicyService ??
new SourceRatePolicyService({ aud0BaseUrl: opts?.aud0BaseUrl, sourceService: this.sources });
}
// -------------------------------------------------------------------------
// source.register
// -------------------------------------------------------------------------
async registerSource(input: RegisterSourceInput): Promise<SourceDescriptor> {
const { rights, provenance, ...base } = input;
const source = await this.sources.createSource(base);
if (rights) {
await this.setRights({ ...rights, source_id: source.source_id, tenant_id: source.tenant_id });
}
if (provenance) {
await this.setProvenance({
...provenance,
source_id: source.source_id,
tenant_id: source.tenant_id,
updated_at: new Date().toISOString(),
});
}
return this.describeSource(source.source_id, source.tenant_id);
}
// -------------------------------------------------------------------------
// rights / license / purpose metadata
// -------------------------------------------------------------------------
async setRights(rights: SourceRights): Promise<SourceRights> {
await this.assertSourceTenant(rights.source_id, rights.tenant_id);
await this.repo.setRights(rights);
await this.emit("0pt.source_rights_set", rights.tenant_id, rights.source_id, {
license_id: rights.license_id,
allowed_purposes: rights.allowed_purposes,
redistribution: rights.redistribution,
});
return rights;
}
async getRights(sourceId: string, tenantId: string): Promise<SourceRights | undefined> {
await this.assertSourceTenant(sourceId, tenantId);
return this.repo.getRights(sourceId);
}
/**
* The purpose gate other capabilities call before using a source. FAIL
* CLOSED: no rights record, or a purpose outside the allowlist, throws
* AUTHORIZATION_DENIED.
*/
async assertPurposeAllowed(sourceId: string, tenantId: string, purpose: string): Promise<SourceRights> {
await this.assertSourceTenant(sourceId, tenantId);
const rights = await this.repo.getRights(sourceId);
if (!rights) {
throw new SiaxError(
"AUTHORIZATION_DENIED",
"0pt.source_registry",
`source ${sourceId} has no rights record — purpose "${purpose}" denied (fail closed)`,
);
}
if (!rights.allowed_purposes.includes(purpose)) {
throw new SiaxError(
"AUTHORIZATION_DENIED",
"0pt.source_registry",
`purpose "${purpose}" is not in source ${sourceId}'s allowed purposes`,
);
}
return rights;
}
// -------------------------------------------------------------------------
// provenance / freshness
// -------------------------------------------------------------------------
async setProvenance(prov: SourceProvenance): Promise<SourceProvenance> {
await this.assertSourceTenant(prov.source_id, prov.tenant_id);
await this.repo.setProvenance(prov);
await this.emit("0pt.source_provenance_set", prov.tenant_id, prov.source_id, {
method: prov.method,
authority: prov.authority,
cadence: prov.cadence,
});
return prov;
}
async getProvenance(sourceId: string, tenantId: string): Promise<SourceProvenance | undefined> {
await this.assertSourceTenant(sourceId, tenantId);
return this.repo.getProvenance(sourceId);
}
/** Record a successful data observation (drives freshness). */
async recordSuccess(sourceId: string, tenantId: string, atIso?: string): Promise<void> {
await this.assertSourceTenant(sourceId, tenantId);
await this.repo.recordSuccess(sourceId, tenantId, atIso ?? new Date().toISOString());
}
async getFreshness(sourceId: string, tenantId: string, now: Date = new Date()): Promise<SourceFreshness> {
await this.assertSourceTenant(sourceId, tenantId);
const prov = await this.repo.getProvenance(sourceId);
const last = await this.repo.getLastSuccess(sourceId);
const sla = prov?.freshness_sla_hours;
if (!last) {
return { source_id: sourceId, freshness_sla_hours: sla, fresh: sla === undefined };
}
const ageHours = (now.getTime() - new Date(last).getTime()) / 3_600_000;
return {
source_id: sourceId,
last_success_at: last,
freshness_sla_hours: sla,
age_hours: Math.round(ageHours * 100) / 100,
fresh: sla === undefined ? true : ageHours <= sla,
};
}
// -------------------------------------------------------------------------
// source.health
// -------------------------------------------------------------------------
async recordHealthCheck(check: SourceHealthCheck): Promise<SourceHealth> {
await this.assertSourceTenant(check.source_id, check.tenant_id);
await this.repo.appendHealthCheck(check);
if (check.status === "up") {
await this.recordSuccess(check.source_id, check.tenant_id, check.checked_at);
}
return this.getHealth(check.source_id, check.tenant_id);
}
async getHealth(sourceId: string, tenantId: string): Promise<SourceHealth> {
await this.assertSourceTenant(sourceId, tenantId);
// Newest-first (repository contract) — index 0 is the most recent check.
const list = await this.repo.listHealthChecks(sourceId);
const last = list[0];
let consecutiveFailures = 0;
for (const check of list) {
if (check.status === "down" || check.status === "degraded") consecutiveFailures++;
else break;
}
const status: SourceHealthStatus = last?.status ?? "unknown";
return { source_id: sourceId, status, last_check: last, consecutive_failures: consecutiveFailures };
}
// -------------------------------------------------------------------------
// source.query / source.describe
// -------------------------------------------------------------------------
async querySources(filter: {
tenant_id: string;
project_id?: string;
source_type?: Source["source_type"];
status?: Source["status"];
/** Only sources whose rights allowlist includes this purpose. */
purpose?: string;
/** Only sources with at most this redistribution openness. */
redistribution?: SourceRights["redistribution"];
}): Promise<Source[]> {
if (!filter.tenant_id) {
throw new SiaxError("INVALID_REQUEST", "0pt.source_registry", "tenant_id is required on source.query");
}
let results = await this.sources.listSources({
tenant_id: filter.tenant_id,
project_id: filter.project_id,
source_type: filter.source_type,
status: filter.status,
});
if (filter.purpose !== undefined || filter.redistribution !== undefined) {
const rightsEntries = await Promise.all(
results.map(async (s) => [s.source_id, await this.repo.getRights(s.source_id)] as const),
);
const rightsBySource = new Map(rightsEntries);
if (filter.purpose !== undefined) {
results = results.filter((s) => {
const rights = rightsBySource.get(s.source_id);
return rights !== undefined && rights.allowed_purposes.includes(filter.purpose as string);
});
}
if (filter.redistribution !== undefined) {
results = results.filter(
(s) => rightsBySource.get(s.source_id)?.redistribution === filter.redistribution,
);
}
}
return results;
}
async describeSource(sourceId: string, tenantId: string): Promise<SourceDescriptor> {
const source = await this.assertSourceTenant(sourceId, tenantId);
const [policies, rights, provenance, health, freshness] = await Promise.all([
this.ratePolicies.listSourceRatePolicies({ tenant_id: tenantId, source_id: sourceId }),
this.repo.getRights(sourceId),
this.repo.getProvenance(sourceId),
this.getHealth(sourceId, tenantId),
this.getFreshness(sourceId, tenantId),
]);
return {
source,
rights,
provenance,
health,
freshness,
rate_policy: policies[0],
};
}
// -------------------------------------------------------------------------
// internals
// -------------------------------------------------------------------------
/**
* Tenant gate for every read/write: resolves the source AND verifies the
* caller's tenant owns it. Cross-tenant probes get NOT_FOUND (no existence
* oracle), same 404-not-403 posture as ST0RE/N0D.
*/
private async assertSourceTenant(sourceId: string, tenantId: string): Promise<Source> {
if (!tenantId) {
throw new SiaxError("INVALID_REQUEST", "0pt.source_registry", "tenant_id is required");
}
let source: Source;
try {
source = await this.sources.getSource(sourceId);
} catch {
throw new SiaxError("NOT_FOUND", "0pt.source_registry", `source ${sourceId} not found`);
}
if (source.tenant_id !== tenantId) {
throw new SiaxError("NOT_FOUND", "0pt.source_registry", `source ${sourceId} not found`);
}
return source;
}
private async emit(
eventType: string,
tenantId: string,
sourceId: string,
payload: Record<string, unknown>,
): Promise<void> {
await this.aud0.ingestEvent({
event_id: `evt_${eventType}_${sourceId}_${Date.now()}`,
event_type: eventType,
tenant_id: tenantId,
project_id: "p_platform",
actor_id: "svc_0pt",
app_id: "0pt",
capability_id: eventType.replace(/^0pt\./, "0pt."),
resource_type: "source",
resource_id: sourceId,
risk_level: "L2",
payload,
created_at: new Date().toISOString(),
});
}
}