Replaces the in-memory (per-instance, not even per-process-shared) Maps in SourceService/SourceRegistryService with real Postgres persistence, and adds the register/query/describe/health/rights/provenance/freshness HTTP routes that never existed (grep for SourceRegistryService in app/ previously returned nothing). - sql/002_source_registry.sql: opt_sources, opt_source_rights, opt_source_provenance, opt_source_health_checks, opt_source_freshness. Numbered 002 (not 006) because 004's opt_source_rate_policies already forward-references opt_sources, which no sql/ file defined until now — fixes 004's pre-existing broken migration ordering (opt_datasets remains a separate, unrelated, still-open gap in 004, flagged not fixed). - src/repositories/*: SourceRepository / SourceRegistryRepository seams, each with an in-memory implementation (business logic extracted verbatim) and a Postgres implementation, chosen via resolveStoreMode — production still fails hard with no DATABASE_URL (Master-DoD §3), but a configured production now actually works instead of always being blocked. - SourceService / SourceRegistryService: storage moved behind the repository seam; getHealth/getFreshness are now async and tenant-scoped at the accessor itself (needed once they're reachable directly over HTTP, not only after an earlier caller's tenant check). - src/services/source-registry.ts: shared singleton instances for the sources/* route group, fixing the "each route module owned its own private registry" defect (source-rate-policies and .../query previously couldn't see a source registered via POST /api/0pt/sources). - New routes: POST/GET /api/0pt/sources, GET /api/0pt/sources/[id], GET+PUT .../rights, GET+PUT .../provenance, GET+POST .../health, GET .../freshness. - Business logic unchanged: fail-closed purpose gate, cross-tenant NOT_FOUND (no existence oracle, §50), bounded 100-row health history. Tested against a real Postgres container (not mocked) — tests/p0/source-registry-postgres.realinfra.test.ts, env-gated on SOURCE_REGISTRY_TEST_DATABASE_URL, skips honestly when unset. All 249 pre-existing P0 tests + 7 new realinfra tests pass; also smoke-tested the full HTTP route surface end-to-end against next dev + live Postgres. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
328 lines
13 KiB
TypeScript
328 lines
13 KiB
TypeScript
/**
|
|
* src/services/source-registry-service.ts
|
|
* ========================================
|
|
* The P0 Source Registry capability surface (Master-DoD §10):
|
|
*
|
|
* source.register -> delegates to SourceService.createSource (owner of
|
|
* 0pt.source.schema.v1) + optional rights/provenance
|
|
* source.query -> listSources + rights/purpose filtering
|
|
* source.describe -> full SourceDescriptor (base + rights + provenance
|
|
* + health + freshness + rate policy)
|
|
* source.health -> record + aggregate health observations
|
|
* source.rate-policy -> already owned by SourceRatePolicyService; composed
|
|
*
|
|
* Tenant isolation: every read path REQUIRES a tenant_id and filters on it —
|
|
* a source registered by tenant A is invisible to tenant B (tested in
|
|
* tests/p0/source-registry.test.ts, incl. the cross-tenant attack case §50).
|
|
* `getHealth`/`getFreshness` now take tenant_id directly and run the same
|
|
* `assertSourceTenant` gate every other accessor does — they are exposed as
|
|
* standalone HTTP routes now (app/api/0pt/sources/[sourceId]/health,
|
|
* .../freshness), so they can no longer rely on an earlier caller having
|
|
* already checked tenancy on their behalf.
|
|
*
|
|
* Purpose enforcement: `assertPurposeAllowed` is the single hook other 0pt
|
|
* capabilities call before using a source for a given purpose (SearchService
|
|
* and DatasetService both do). No rights record ⇒ fail closed (deny).
|
|
*
|
|
* Persistence: rights/provenance/health/freshness now live behind
|
|
* SourceRegistryRepository (src/repositories/source-registry-repository.ts)
|
|
* — real Postgres (`opt_source_rights`/`opt_source_provenance`/
|
|
* `opt_source_health_checks`/`opt_source_freshness`,
|
|
* sql/002_source_registry.sql) when DATABASE_URL is configured, in-memory
|
|
* Maps otherwise. `resolveStoreMode` still fails production boot hard with
|
|
* no DATABASE_URL (Master-DoD §3) — the difference from before is that a
|
|
* configured production now actually WORKS (opt_sources/etc. exist), not
|
|
* just fails safely.
|
|
*/
|
|
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
|
import { SiaxError } from "../platform/errors";
|
|
import {
|
|
createSourceRegistryRepository,
|
|
type SourceRegistryRepository,
|
|
} from "../repositories/source-registry-repository";
|
|
import { SourceService, type SourceServiceOptions } from "./source-service";
|
|
import { SourceRatePolicyService } from "./source-rate-policy-service";
|
|
import type { CreateSourceInput, Source } from "../types/source";
|
|
import type {
|
|
SourceDescriptor,
|
|
SourceFreshness,
|
|
SourceHealth,
|
|
SourceHealthCheck,
|
|
SourceHealthStatus,
|
|
SourceProvenance,
|
|
SourceRights,
|
|
} from "../types/source-registry";
|
|
|
|
export interface RegisterSourceInput extends CreateSourceInput {
|
|
rights?: Omit<SourceRights, "source_id" | "tenant_id">;
|
|
provenance?: Omit<SourceProvenance, "source_id" | "tenant_id" | "updated_at">;
|
|
}
|
|
|
|
export class SourceRegistryService {
|
|
private readonly repo: SourceRegistryRepository;
|
|
private readonly aud0: Aud0Client;
|
|
readonly sources: SourceService;
|
|
readonly ratePolicies: SourceRatePolicyService;
|
|
|
|
constructor(opts?: {
|
|
aud0BaseUrl?: string;
|
|
/** Explicit DATABASE_URL override — mainly for tests. Falls back to resolveStoreMode's own env lookup when omitted. */
|
|
databaseUrl?: string;
|
|
sourceService?: SourceService;
|
|
ratePolicyService?: SourceRatePolicyService;
|
|
/** Inject a repository directly (tests) — bypasses resolveStoreMode entirely. */
|
|
repository?: SourceRegistryRepository;
|
|
}) {
|
|
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
|
this.repo = opts?.repository ?? createSourceRegistryRepository(opts?.databaseUrl);
|
|
const sourceServiceOpts: SourceServiceOptions = {
|
|
aud0BaseUrl: opts?.aud0BaseUrl,
|
|
databaseUrl: opts?.databaseUrl,
|
|
};
|
|
this.sources = opts?.sourceService ?? new SourceService(sourceServiceOpts);
|
|
this.ratePolicies =
|
|
opts?.ratePolicyService ??
|
|
new SourceRatePolicyService({ aud0BaseUrl: opts?.aud0BaseUrl, sourceService: this.sources });
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// source.register
|
|
// -------------------------------------------------------------------------
|
|
async registerSource(input: RegisterSourceInput): Promise<SourceDescriptor> {
|
|
const { rights, provenance, ...base } = input;
|
|
const source = await this.sources.createSource(base);
|
|
if (rights) {
|
|
await this.setRights({ ...rights, source_id: source.source_id, tenant_id: source.tenant_id });
|
|
}
|
|
if (provenance) {
|
|
await this.setProvenance({
|
|
...provenance,
|
|
source_id: source.source_id,
|
|
tenant_id: source.tenant_id,
|
|
updated_at: new Date().toISOString(),
|
|
});
|
|
}
|
|
return this.describeSource(source.source_id, source.tenant_id);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// rights / license / purpose metadata
|
|
// -------------------------------------------------------------------------
|
|
async setRights(rights: SourceRights): Promise<SourceRights> {
|
|
await this.assertSourceTenant(rights.source_id, rights.tenant_id);
|
|
await this.repo.setRights(rights);
|
|
await this.emit("0pt.source_rights_set", rights.tenant_id, rights.source_id, {
|
|
license_id: rights.license_id,
|
|
allowed_purposes: rights.allowed_purposes,
|
|
redistribution: rights.redistribution,
|
|
});
|
|
return rights;
|
|
}
|
|
|
|
async getRights(sourceId: string, tenantId: string): Promise<SourceRights | undefined> {
|
|
await this.assertSourceTenant(sourceId, tenantId);
|
|
return this.repo.getRights(sourceId);
|
|
}
|
|
|
|
/**
|
|
* The purpose gate other capabilities call before using a source. FAIL
|
|
* CLOSED: no rights record, or a purpose outside the allowlist, throws
|
|
* AUTHORIZATION_DENIED.
|
|
*/
|
|
async assertPurposeAllowed(sourceId: string, tenantId: string, purpose: string): Promise<SourceRights> {
|
|
await this.assertSourceTenant(sourceId, tenantId);
|
|
const rights = await this.repo.getRights(sourceId);
|
|
if (!rights) {
|
|
throw new SiaxError(
|
|
"AUTHORIZATION_DENIED",
|
|
"0pt.source_registry",
|
|
`source ${sourceId} has no rights record — purpose "${purpose}" denied (fail closed)`,
|
|
);
|
|
}
|
|
if (!rights.allowed_purposes.includes(purpose)) {
|
|
throw new SiaxError(
|
|
"AUTHORIZATION_DENIED",
|
|
"0pt.source_registry",
|
|
`purpose "${purpose}" is not in source ${sourceId}'s allowed purposes`,
|
|
);
|
|
}
|
|
return rights;
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// provenance / freshness
|
|
// -------------------------------------------------------------------------
|
|
async setProvenance(prov: SourceProvenance): Promise<SourceProvenance> {
|
|
await this.assertSourceTenant(prov.source_id, prov.tenant_id);
|
|
await this.repo.setProvenance(prov);
|
|
await this.emit("0pt.source_provenance_set", prov.tenant_id, prov.source_id, {
|
|
method: prov.method,
|
|
authority: prov.authority,
|
|
cadence: prov.cadence,
|
|
});
|
|
return prov;
|
|
}
|
|
|
|
async getProvenance(sourceId: string, tenantId: string): Promise<SourceProvenance | undefined> {
|
|
await this.assertSourceTenant(sourceId, tenantId);
|
|
return this.repo.getProvenance(sourceId);
|
|
}
|
|
|
|
/** Record a successful data observation (drives freshness). */
|
|
async recordSuccess(sourceId: string, tenantId: string, atIso?: string): Promise<void> {
|
|
await this.assertSourceTenant(sourceId, tenantId);
|
|
await this.repo.recordSuccess(sourceId, tenantId, atIso ?? new Date().toISOString());
|
|
}
|
|
|
|
async getFreshness(sourceId: string, tenantId: string, now: Date = new Date()): Promise<SourceFreshness> {
|
|
await this.assertSourceTenant(sourceId, tenantId);
|
|
const prov = await this.repo.getProvenance(sourceId);
|
|
const last = await this.repo.getLastSuccess(sourceId);
|
|
const sla = prov?.freshness_sla_hours;
|
|
if (!last) {
|
|
return { source_id: sourceId, freshness_sla_hours: sla, fresh: sla === undefined };
|
|
}
|
|
const ageHours = (now.getTime() - new Date(last).getTime()) / 3_600_000;
|
|
return {
|
|
source_id: sourceId,
|
|
last_success_at: last,
|
|
freshness_sla_hours: sla,
|
|
age_hours: Math.round(ageHours * 100) / 100,
|
|
fresh: sla === undefined ? true : ageHours <= sla,
|
|
};
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// source.health
|
|
// -------------------------------------------------------------------------
|
|
async recordHealthCheck(check: SourceHealthCheck): Promise<SourceHealth> {
|
|
await this.assertSourceTenant(check.source_id, check.tenant_id);
|
|
await this.repo.appendHealthCheck(check);
|
|
if (check.status === "up") {
|
|
await this.recordSuccess(check.source_id, check.tenant_id, check.checked_at);
|
|
}
|
|
return this.getHealth(check.source_id, check.tenant_id);
|
|
}
|
|
|
|
async getHealth(sourceId: string, tenantId: string): Promise<SourceHealth> {
|
|
await this.assertSourceTenant(sourceId, tenantId);
|
|
// Newest-first (repository contract) — index 0 is the most recent check.
|
|
const list = await this.repo.listHealthChecks(sourceId);
|
|
const last = list[0];
|
|
let consecutiveFailures = 0;
|
|
for (const check of list) {
|
|
if (check.status === "down" || check.status === "degraded") consecutiveFailures++;
|
|
else break;
|
|
}
|
|
const status: SourceHealthStatus = last?.status ?? "unknown";
|
|
return { source_id: sourceId, status, last_check: last, consecutive_failures: consecutiveFailures };
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// source.query / source.describe
|
|
// -------------------------------------------------------------------------
|
|
async querySources(filter: {
|
|
tenant_id: string;
|
|
project_id?: string;
|
|
source_type?: Source["source_type"];
|
|
status?: Source["status"];
|
|
/** Only sources whose rights allowlist includes this purpose. */
|
|
purpose?: string;
|
|
/** Only sources with at most this redistribution openness. */
|
|
redistribution?: SourceRights["redistribution"];
|
|
}): Promise<Source[]> {
|
|
if (!filter.tenant_id) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.source_registry", "tenant_id is required on source.query");
|
|
}
|
|
let results = await this.sources.listSources({
|
|
tenant_id: filter.tenant_id,
|
|
project_id: filter.project_id,
|
|
source_type: filter.source_type,
|
|
status: filter.status,
|
|
});
|
|
if (filter.purpose !== undefined || filter.redistribution !== undefined) {
|
|
const rightsEntries = await Promise.all(
|
|
results.map(async (s) => [s.source_id, await this.repo.getRights(s.source_id)] as const),
|
|
);
|
|
const rightsBySource = new Map(rightsEntries);
|
|
if (filter.purpose !== undefined) {
|
|
results = results.filter((s) => {
|
|
const rights = rightsBySource.get(s.source_id);
|
|
return rights !== undefined && rights.allowed_purposes.includes(filter.purpose as string);
|
|
});
|
|
}
|
|
if (filter.redistribution !== undefined) {
|
|
results = results.filter(
|
|
(s) => rightsBySource.get(s.source_id)?.redistribution === filter.redistribution,
|
|
);
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
|
|
async describeSource(sourceId: string, tenantId: string): Promise<SourceDescriptor> {
|
|
const source = await this.assertSourceTenant(sourceId, tenantId);
|
|
const [policies, rights, provenance, health, freshness] = await Promise.all([
|
|
this.ratePolicies.listSourceRatePolicies({ tenant_id: tenantId, source_id: sourceId }),
|
|
this.repo.getRights(sourceId),
|
|
this.repo.getProvenance(sourceId),
|
|
this.getHealth(sourceId, tenantId),
|
|
this.getFreshness(sourceId, tenantId),
|
|
]);
|
|
return {
|
|
source,
|
|
rights,
|
|
provenance,
|
|
health,
|
|
freshness,
|
|
rate_policy: policies[0],
|
|
};
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// internals
|
|
// -------------------------------------------------------------------------
|
|
/**
|
|
* Tenant gate for every read/write: resolves the source AND verifies the
|
|
* caller's tenant owns it. Cross-tenant probes get NOT_FOUND (no existence
|
|
* oracle), same 404-not-403 posture as ST0RE/N0D.
|
|
*/
|
|
private async assertSourceTenant(sourceId: string, tenantId: string): Promise<Source> {
|
|
if (!tenantId) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.source_registry", "tenant_id is required");
|
|
}
|
|
let source: Source;
|
|
try {
|
|
source = await this.sources.getSource(sourceId);
|
|
} catch {
|
|
throw new SiaxError("NOT_FOUND", "0pt.source_registry", `source ${sourceId} not found`);
|
|
}
|
|
if (source.tenant_id !== tenantId) {
|
|
throw new SiaxError("NOT_FOUND", "0pt.source_registry", `source ${sourceId} not found`);
|
|
}
|
|
return source;
|
|
}
|
|
|
|
private async emit(
|
|
eventType: string,
|
|
tenantId: string,
|
|
sourceId: string,
|
|
payload: Record<string, unknown>,
|
|
): Promise<void> {
|
|
await this.aud0.ingestEvent({
|
|
event_id: `evt_${eventType}_${sourceId}_${Date.now()}`,
|
|
event_type: eventType,
|
|
tenant_id: tenantId,
|
|
project_id: "p_platform",
|
|
actor_id: "svc_0pt",
|
|
app_id: "0pt",
|
|
capability_id: eventType.replace(/^0pt\./, "0pt."),
|
|
resource_type: "source",
|
|
resource_id: sourceId,
|
|
risk_level: "L2",
|
|
payload,
|
|
created_at: new Date().toISOString(),
|
|
});
|
|
}
|
|
}
|