feat(data-os): Source Registry Postgres persistence + full HTTP route surface

Replaces the in-memory (per-instance, not even per-process-shared) Maps in
SourceService/SourceRegistryService with real Postgres persistence, and adds
the register/query/describe/health/rights/provenance/freshness HTTP routes
that never existed (grep for SourceRegistryService in app/ previously
returned nothing).

- sql/002_source_registry.sql: opt_sources, opt_source_rights,
  opt_source_provenance, opt_source_health_checks, opt_source_freshness.
  Numbered 002 (not 006) because 004's opt_source_rate_policies already
  forward-references opt_sources, which no sql/ file defined until now —
  fixes 004's pre-existing broken migration ordering (opt_datasets remains
  a separate, unrelated, still-open gap in 004, flagged not fixed).
- src/repositories/*: SourceRepository / SourceRegistryRepository seams,
  each with an in-memory implementation (business logic extracted verbatim)
  and a Postgres implementation, chosen via resolveStoreMode — production
  still fails hard with no DATABASE_URL (Master-DoD §3), but a configured
  production now actually works instead of always being blocked.
- SourceService / SourceRegistryService: storage moved behind the
  repository seam; getHealth/getFreshness are now async and tenant-scoped
  at the accessor itself (needed once they're reachable directly over HTTP,
  not only after an earlier caller's tenant check).
- src/services/source-registry.ts: shared singleton instances for the
  sources/* route group, fixing the "each route module owned its own
  private registry" defect (source-rate-policies and .../query previously
  couldn't see a source registered via POST /api/0pt/sources).
- New routes: POST/GET /api/0pt/sources, GET /api/0pt/sources/[id],
  GET+PUT .../rights, GET+PUT .../provenance, GET+POST .../health,
  GET .../freshness.
- Business logic unchanged: fail-closed purpose gate, cross-tenant
  NOT_FOUND (no existence oracle, §50), bounded 100-row health history.

Tested against a real Postgres container (not mocked) —
tests/p0/source-registry-postgres.realinfra.test.ts, env-gated on
SOURCE_REGISTRY_TEST_DATABASE_URL, skips honestly when unset. All 249
pre-existing P0 tests + 7 new realinfra tests pass; also smoke-tested the
full HTTP route surface end-to-end against next dev + live Postgres.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Simon Axelsson
2026-09-01 20:42:41 +02:00
co-authored by Claude Sonnet 5
parent f8bb0d5ab9
commit 07b13bd833
26 changed files with 1578 additions and 177 deletions
@@ -0,0 +1,27 @@
/**
* app/api/0pt/sources/[sourceId]/freshness/route.ts
* =====================================================
* Freshness view derived from provenance's freshness_sla_hours + the last
* successful observation (recorded via POST .../health with status "up",
* or SourceRegistryService.recordSuccess directly). Read-only — there is no
* separate write path here by design; freshness is a derived signal, not
* its own record.
*/
import { NextRequest, NextResponse } from "next/server";
import { sourceRegistryService } from "@/src/services/source-registry";
import { siaxErrorResponse } from "@/src/http/siax-error-response";
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
try {
const { sourceId } = await params;
const { searchParams } = new URL(request.url);
const tenantId = searchParams.get("tenant_id");
if (!tenantId) {
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
}
const freshness = await sourceRegistryService.getFreshness(sourceId, tenantId);
return NextResponse.json(freshness);
} catch (error: unknown) {
return siaxErrorResponse(error);
}
}
@@ -0,0 +1,50 @@
/**
* app/api/0pt/sources/[sourceId]/health/route.ts
* ==================================================
* source.health — record a health observation (POST) and read the
* aggregate view (GET): status, last check, consecutive failures. Bounded
* history (latest 100 observations) — this is an operational signal, not
* an audit log; AUD0 owns history.
*/
import { NextRequest, NextResponse } from "next/server";
import { parseBody, recordSourceHealthCheckSchema } from "@/src/validation/0pt-schemas";
import { sourceRegistryService } from "@/src/services/source-registry";
import { siaxErrorResponse } from "@/src/http/siax-error-response";
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
try {
const { sourceId } = await params;
const { searchParams } = new URL(request.url);
const tenantId = searchParams.get("tenant_id");
if (!tenantId) {
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
}
const health = await sourceRegistryService.getHealth(sourceId, tenantId);
return NextResponse.json(health);
} catch (error: unknown) {
return siaxErrorResponse(error);
}
}
export async function POST(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
try {
const { sourceId } = await params;
const __raw = await request.json();
const __parsed = parseBody(recordSourceHealthCheckSchema, __raw);
if (!__parsed.ok) {
return NextResponse.json(__parsed.body, { status: __parsed.status });
}
const body = __parsed.data;
const health = await sourceRegistryService.recordHealthCheck({
source_id: sourceId,
tenant_id: body.tenant_id,
status: body.status,
checked_at: body.checked_at ?? new Date().toISOString(),
latency_ms: body.latency_ms,
detail: body.detail,
});
return NextResponse.json(health, { status: 201 });
} catch (error: unknown) {
return siaxErrorResponse(error);
}
}
@@ -0,0 +1,52 @@
/**
* app/api/0pt/sources/[sourceId]/provenance/route.ts
* ======================================================
* Per-SOURCE provenance/method metadata (Master-DoD §10) — HOW a source is
* obtained (official_api | bulk_download | authorized_web | licensed_feed |
* manual_upload), not per-field bridge_provenance. See BEFORE_STATE.md §3
* for that distinction: this route does not, and cannot, close the
* bridge_provenance gap — it only surfaces what SourceRegistryService
* already models at the source level.
*/
import { NextRequest, NextResponse } from "next/server";
import { parseBody, setSourceProvenanceSchema } from "@/src/validation/0pt-schemas";
import { sourceRegistryService } from "@/src/services/source-registry";
import { siaxErrorResponse } from "@/src/http/siax-error-response";
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
try {
const { sourceId } = await params;
const { searchParams } = new URL(request.url);
const tenantId = searchParams.get("tenant_id");
if (!tenantId) {
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
}
const provenance = await sourceRegistryService.getProvenance(sourceId, tenantId);
if (!provenance) {
return NextResponse.json({ error: `source ${sourceId} has no provenance record` }, { status: 404 });
}
return NextResponse.json(provenance);
} catch (error: unknown) {
return siaxErrorResponse(error);
}
}
export async function PUT(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
try {
const { sourceId } = await params;
const __raw = await request.json();
const __parsed = parseBody(setSourceProvenanceSchema, __raw);
if (!__parsed.ok) {
return NextResponse.json(__parsed.body, { status: __parsed.status });
}
const body = __parsed.data;
const provenance = await sourceRegistryService.setProvenance({
...body,
source_id: sourceId,
updated_at: new Date().toISOString(),
});
return NextResponse.json(provenance);
} catch (error: unknown) {
return siaxErrorResponse(error);
}
}
+14 -17
View File
@@ -4,27 +4,24 @@
* 0pt.query_source — bounded synchronous query execution for a registered
* Source. Additive: does not modify sibling ../route.ts or ../../route.ts.
*
* NOTE — in-memory scope: this module instantiates its own SourceService,
* matching the existing sources/route.ts and sources/[sourceId]/route.ts
* pattern (each API route module owns a fresh in-memory registry, so state
* is not shared across route modules yet — see the "Singleton note" in
* src/services/document-capture-service.ts for prior art on this same repo
* limitation). A source registered via POST /api/0pt/sources will not be
* resolvable here until SourceService is backed by shared persistence.
* Correction: this comment previously claimed `sql/002_p0_dataset_quality.sql`
* already defined an `opt_sources` table for that. That file does not exist
* and never has (git log --all: zero commits, any branch, ever). sql/ today
* has only 001_init.sql and 004_iceberg_write_boundary.sql. The real fix —
* backing SourceService with real persistence — is still open; no migration
* for it exists yet.
* Details: docs/P0_QUERY_SOURCE_FLOW.md.
* UPDATED (Source Registry Postgres-persistence pass): now imports the
* shared `sourceService` singleton from src/services/source-registry.ts
* instead of constructing its own SourceService, closing the "not even a
* shared process-wide singleton" gap BEFORE_STATE.md §1 flagged (a source
* registered via POST /api/0pt/sources used to be invisible here). With
* DATABASE_URL configured this is doubly fixed — state lives in Postgres
* (sql/002_source_registry.sql), genuinely shared across processes, not
* just within one. Details: docs/P0_QUERY_SOURCE_FLOW.md.
*/
import { NextRequest, NextResponse } from "next/server";
import { parseBody, sourceQuerySchema } from "@/src/validation/0pt-schemas";
import { SourceService } from "@/src/services/source-service";
import { SourceQueryService, SourceQueryTimeoutError, SourceQueryUnsupportedError } from "@/src/services/source-query-service";
import { sourceService } from "@/src/services/source-registry";
import {
SourceQueryService,
SourceQueryTimeoutError,
SourceQueryUnsupportedError,
} from "@/src/services/source-query-service";
const sourceService = new SourceService();
const queryService = new SourceQueryService({ sourceService });
export async function POST(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
@@ -0,0 +1,45 @@
/**
* app/api/0pt/sources/[sourceId]/rights/route.ts
* ==================================================
* Rights / license / purpose metadata (Master-DoD §10). PUT is an upsert
* (SourceRegistryService.setRights), matching the fail-closed purpose gate:
* no rights record at all denies every purpose until one is set here.
*/
import { NextRequest, NextResponse } from "next/server";
import { parseBody, setSourceRightsSchema } from "@/src/validation/0pt-schemas";
import { sourceRegistryService } from "@/src/services/source-registry";
import { siaxErrorResponse } from "@/src/http/siax-error-response";
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
try {
const { sourceId } = await params;
const { searchParams } = new URL(request.url);
const tenantId = searchParams.get("tenant_id");
if (!tenantId) {
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
}
const rights = await sourceRegistryService.getRights(sourceId, tenantId);
if (!rights) {
return NextResponse.json({ error: `source ${sourceId} has no rights record` }, { status: 404 });
}
return NextResponse.json(rights);
} catch (error: unknown) {
return siaxErrorResponse(error);
}
}
export async function PUT(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
try {
const { sourceId } = await params;
const __raw = await request.json();
const __parsed = parseBody(setSourceRightsSchema, __raw);
if (!__parsed.ok) {
return NextResponse.json(__parsed.body, { status: __parsed.status });
}
const body = __parsed.data;
const rights = await sourceRegistryService.setRights({ ...body, source_id: sourceId });
return NextResponse.json(rights);
} catch (error: unknown) {
return siaxErrorResponse(error);
}
}
+26
View File
@@ -0,0 +1,26 @@
/**
* app/api/0pt/sources/[sourceId]/route.ts
* ==========================================
* source.describe — full SourceDescriptor (base + rights + provenance +
* health + freshness + rate policy). Cross-tenant probes get 404 NOT_FOUND
* (no existence oracle, §50) via SourceRegistryService.describeSource's
* assertSourceTenant gate.
*/
import { NextRequest, NextResponse } from "next/server";
import { sourceRegistryService } from "@/src/services/source-registry";
import { siaxErrorResponse } from "@/src/http/siax-error-response";
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
try {
const { sourceId } = await params;
const { searchParams } = new URL(request.url);
const tenantId = searchParams.get("tenant_id");
if (!tenantId) {
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
}
const descriptor = await sourceRegistryService.describeSource(sourceId, tenantId);
return NextResponse.json(descriptor);
} catch (error: unknown) {
return siaxErrorResponse(error);
}
}
@@ -1,9 +1,10 @@
import { NextResponse } from "next/server";
import { SourceRatePolicyService } from "@/src/services/source-rate-policy-service";
import { sourceRatePolicyService as service } from "@/src/services/source-registry";
const service = new SourceRatePolicyService();
export async function GET(_request: Request, { params }: { params: Promise<{ sourceRatePolicyId: string }> }) {
export async function GET(
_request: Request,
{ params }: { params: Promise<{ sourceRatePolicyId: string }> },
) {
try {
const { sourceRatePolicyId } = await params;
const policy = await service.getSourceRatePolicy(sourceRatePolicyId);
+1 -3
View File
@@ -1,8 +1,6 @@
import { NextRequest, NextResponse } from "next/server";
import { parseBody, createSourceRatePolicySchema } from "@/src/validation/0pt-schemas";
import { SourceRatePolicyService } from "@/src/services/source-rate-policy-service";
const service = new SourceRatePolicyService();
import { sourceRatePolicyService as service } from "@/src/services/source-registry";
export async function POST(request: NextRequest) {
try {
+55
View File
@@ -0,0 +1,55 @@
/**
* app/api/0pt/sources/route.ts
* ==============================
* source.register (POST) + source.query (GET) — the P0 Source Registry
* route surface BEFORE_STATE.md §1 found entirely missing
* (`grep -rl "SourceRegistryService" app/` returned nothing). Backed by
* SourceRegistryService, which now persists to Postgres when DATABASE_URL
* is configured (src/repositories/*, sql/002_source_registry.sql).
*
* Uses the shared singleton from src/services/source-registry.ts, not a
* fresh `new SourceRegistryService()` per route module — see that file's
* doc comment.
*/
import { NextRequest, NextResponse } from "next/server";
import { parseBody, registerSourceSchema } from "@/src/validation/0pt-schemas";
import { sourceRegistryService } from "@/src/services/source-registry";
import { siaxErrorResponse } from "@/src/http/siax-error-response";
import type { SourceStatus, SourceType } from "@/src/types/source";
import type { SourceRights } from "@/src/types/source-registry";
export async function POST(request: NextRequest) {
try {
const __raw = await request.json();
const __parsed = parseBody(registerSourceSchema, __raw);
if (!__parsed.ok) {
return NextResponse.json(__parsed.body, { status: __parsed.status });
}
const body = __parsed.data;
const descriptor = await sourceRegistryService.registerSource(body);
return NextResponse.json(descriptor, { status: 201 });
} catch (error: unknown) {
return siaxErrorResponse(error);
}
}
export async function GET(request: NextRequest) {
try {
const { searchParams } = new URL(request.url);
const tenantId = searchParams.get("tenant_id");
if (!tenantId) {
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
}
const sources = await sourceRegistryService.querySources({
tenant_id: tenantId,
project_id: searchParams.get("project_id") ?? undefined,
source_type: (searchParams.get("source_type") as SourceType) ?? undefined,
status: (searchParams.get("status") as SourceStatus) ?? undefined,
purpose: searchParams.get("purpose") ?? undefined,
redistribution: (searchParams.get("redistribution") as SourceRights["redistribution"]) ?? undefined,
});
return NextResponse.json(sources);
} catch (error: unknown) {
return siaxErrorResponse(error);
}
}
+105
View File
@@ -0,0 +1,105 @@
-- 0pt Source Registry persistence (Master-DoD §10; BEFORE_STATE.md §1).
--
-- Backs src/services/source-service.ts (0pt.source.schema.v1, the base
-- registry) and src/services/source-registry-service.ts (rights/license/
-- purpose, provenance, health, freshness — the P0 Source Registry surface
-- layered on top). Both previously held their state in bare `new Map()`s,
-- scoped per service *instance* (not even shared within one process) and
-- hard-blocked from production boot by resolveStoreMode
-- (src/platform/dependency-mode.ts) for lack of a DB. This migration is the
-- DB half of closing that gap; src/repositories/* is the code half.
--
-- Deliberately numbered 002, not 006: sql/004_iceberg_write_boundary.sql's
-- opt_source_rate_policies table already carries
-- `source_id TEXT NOT NULL REFERENCES opt_sources (source_id)` — a
-- forward reference to a table that has never existed in any sql/ file
-- (opt_sources was, until this file, only ever a doc-comment aspiration —
-- see the "What Postgres migration would require" section of
-- BEFORE_STATE.md §1 and the retracted `sql/002_p0_dataset_quality.sql`
-- claim in CLAUDE.md/app/api/0pt/sources/[sourceId]/query/route.ts, an
-- UNRELATED fabricated claim about a different, never-written file that
-- happened to reuse this same number). Applying 004 before opt_sources
-- exists would fail outright on a fresh database; this file must run
-- between 001 and 004 to make 004's existing FK resolvable. This is a
-- pre-existing ordering bug in 004, not introduced here — fixed here only
-- because this pass needed opt_sources to exist anyway.
-- ── 0pt.source.schema.v1 (src/types/source.ts) ─────────────────────────
CREATE TABLE IF NOT EXISTS opt_sources (
source_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
project_id TEXT NOT NULL,
source_type TEXT NOT NULL, -- database | file | web | api | stream | manual
name TEXT NOT NULL,
adapter_id TEXT, -- e.g. "fabric_onelake_sql" (src/adapters/fabric-onelake-adapter.ts)
access TEXT, -- api | bulk | authorized_web | licensed (never a bypass mode)
description TEXT,
status TEXT NOT NULL DEFAULT 'active', -- active | paused | archived | deprecated
correlation_id TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ,
created_by TEXT
);
CREATE INDEX IF NOT EXISTS opt_sources_tenant_project_idx ON opt_sources (tenant_id, project_id);
CREATE INDEX IF NOT EXISTS opt_sources_type_idx ON opt_sources (source_type);
CREATE INDEX IF NOT EXISTS opt_sources_status_idx ON opt_sources (status);
-- ── Rights / license / purpose metadata (src/types/source-registry.ts) ─
-- One row per source (SourceRegistryService.setRights is an upsert-by-
-- source_id, same as the Map it replaces). No rights row = fail-closed
-- deny in assertPurposeAllowed — that logic lives in the service, not here.
CREATE TABLE IF NOT EXISTS opt_source_rights (
source_id TEXT PRIMARY KEY REFERENCES opt_sources (source_id),
tenant_id TEXT NOT NULL,
license_id TEXT NOT NULL,
license_url TEXT,
allowed_purposes TEXT[] NOT NULL DEFAULT '{}',
pii_allowed BOOLEAN NOT NULL DEFAULT false,
redistribution TEXT NOT NULL, -- internal | customer_deliverable | public | prohibited
attribution TEXT,
verified_at TEXT NOT NULL, -- ISO date the assessment was made (date-only in practice; kept as TEXT, not TIMESTAMPTZ, so it round-trips byte-for-byte)
verified_by TEXT NOT NULL,
notes TEXT,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS opt_source_rights_tenant_idx ON opt_source_rights (tenant_id);
-- ── Provenance / method metadata (per-SOURCE, not per-field — see
-- BEFORE_STATE.md §3 for the distinction from bridge_provenance) ────────
CREATE TABLE IF NOT EXISTS opt_source_provenance (
source_id TEXT PRIMARY KEY REFERENCES opt_sources (source_id),
tenant_id TEXT NOT NULL,
method TEXT NOT NULL, -- official_api | bulk_download | authorized_web | licensed_feed | manual_upload
authority TEXT,
compliance_evidence_ref TEXT,
cadence TEXT,
freshness_sla_hours NUMERIC,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS opt_source_provenance_tenant_idx ON opt_source_provenance (tenant_id);
-- ── Health observations (source.health) — an operational signal, NOT an
-- audit log (AUD0 owns history). Bounded to the latest 100 rows per
-- source_id by the repository layer (src/repositories/postgres-source-
-- registry-repository.ts), mirroring the in-memory implementation's
-- `list.splice(0, list.length - 100)` bound.
CREATE TABLE IF NOT EXISTS opt_source_health_checks (
health_check_id BIGSERIAL PRIMARY KEY,
source_id TEXT NOT NULL REFERENCES opt_sources (source_id),
tenant_id TEXT NOT NULL,
status TEXT NOT NULL, -- up | degraded | down | unknown
checked_at TIMESTAMPTZ NOT NULL,
latency_ms INT,
detail TEXT -- short operator-facing text; never a raw upstream body
);
CREATE INDEX IF NOT EXISTS opt_source_health_checks_source_idx ON opt_source_health_checks (source_id, checked_at DESC, health_check_id DESC);
-- ── Freshness: last successful observation per source (source.health's
-- "up" checks call recordSuccess; getFreshness reads this + provenance's
-- freshness_sla_hours). One row per source — upserted, not appended.
CREATE TABLE IF NOT EXISTS opt_source_freshness (
source_id TEXT PRIMARY KEY REFERENCES opt_sources (source_id),
tenant_id TEXT NOT NULL,
last_success_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX IF NOT EXISTS opt_source_freshness_tenant_idx ON opt_source_freshness (tenant_id);
+12
View File
@@ -8,6 +8,18 @@
-- different service's database. 0pt holds only the reference, never a copy
-- of ST0RE's table/catalog/snapshot state (see the module doc comment on
-- src/clients/st0re-lakehouse-client.ts).
--
-- Ordering note (added by the Source Registry Postgres-persistence pass):
-- opt_source_rate_policies below has always forward-referenced
-- `opt_sources (source_id)`, a table no sql/ file defined until
-- sql/002_source_registry.sql landed -- applying this file to a fresh
-- database before 002 would fail on that FK. 002 is now the one that must
-- run first (001 -> 002 -> 004 -> 005). Flagging, not fixing here: this
-- file's OTHER forward reference, `opt_dataset_iceberg_mappings.dataset_id
-- REFERENCES opt_datasets (dataset_id)` below, has the identical problem --
-- no sql/ file defines opt_datasets either -- and remains open; it is a
-- dataset-registry gap, unrelated to and out of scope for the Source
-- Registry work that added 002.
CREATE TABLE IF NOT EXISTS opt_dataset_iceberg_mappings (
dataset_iceberg_mapping_id TEXT PRIMARY KEY,
+25
View File
@@ -0,0 +1,25 @@
/**
* src/http/siax-error-response.ts
* =================================
* Maps a thrown error to a NextResponse for the source-registry routes
* (app/api/0pt/sources/*). SourceRegistryService throws real `SiaxError`
* instances (src/platform/errors.ts) with a pinned `.httpStatus`/`.code` —
* this honors that mapping directly instead of re-deriving status codes from
* regex-matching `error.message`, the ad-hoc pattern most other 0pt routes
* use (app/api/0pt/documents/[documentId]/extraction/route.ts etc.) because
* their services still throw plain `Error`. SourceService/SourceRegistryService's
* lower-level plain Errors ("already exists", "not found") still fall back
* to that same regex convention here, for consistency with the rest of the
* repo's error surface.
*/
import { NextResponse } from "next/server";
import { SiaxError } from "../platform/errors";
export function siaxErrorResponse(error: unknown): NextResponse {
if (error instanceof SiaxError) {
return NextResponse.json(error.toJSON(), { status: error.httpStatus });
}
const message = error instanceof Error ? error.message : "Unknown error";
const status = /already exists/i.test(message) ? 409 : /not found/i.test(message) ? 404 : 400;
return NextResponse.json({ error: message }, { status });
}
+61
View File
@@ -0,0 +1,61 @@
/**
* src/repositories/db.ts
* ========================
* Lazy Postgres connection for P0 capability repositories (src/repositories/*).
*
* Deliberately separate from lib/db.ts: that module is tuned for the
* Vercel-hosted edge Postgres behind app/api/orders + app/api/subscriptions
* (`ssl: "require"`, because that's what Vercel Postgres needs) and lives in
* lib/, which src/ (the blocking-typechecked capability layer,
* .dependency-cruiser.cjs's "no-src-importing-app-or-components" rule's
* sibling concern) does not depend on. The P0 capability layer's Postgres
* instance is the self-hosted VPS Postgres from docs/08-infrastructure.md
* (infra/docker-compose.yml's `postgres` service, same private docker network
* as the FastAPI engine, no TLS terminated in front of it) -- forcing
* `ssl: "require"` here would break every real connection to it. SSL is
* opt-in via PGSSL=require for deployments that do put TLS in front.
*
* Same lazy/env-guarded shape as lib/db.ts, and the same production-safety
* decision as every other P0 dependency: src/platform/dependency-mode.ts's
* `resolveStoreMode` is what actually decides postgres-vs-memory (and fails
* hard in production with no URL) -- this module only opens the connection
* once that decision has already been made.
*/
import postgres from "postgres";
export type RepositorySql = ReturnType<typeof postgres>;
/** DATABASE_URL, falling back to Vercel Postgres's env names -- same precedence lib/db.ts's dbUrl() uses, kept independent so src/ has no import on lib/. */
export function repositoryDatabaseUrl(env: NodeJS.ProcessEnv = process.env): string | undefined {
return env.DATABASE_URL || env.POSTGRES_URL || env.POSTGRES_PRISMA_URL;
}
let _sql: RepositorySql | null = null;
let _urlUsed: string | undefined;
/** Returns a memoized connection for `url`; re-opens if a different URL is requested (e.g. tests pointing at a fresh testcontainer). */
export function getRepositorySql(url: string): RepositorySql {
if (_sql && _urlUsed === url) return _sql;
_sql = postgres(url, {
prepare: false,
idle_timeout: 20,
max: 5,
ssl: process.env.PGSSL === "require" ? "require" : false,
});
_urlUsed = url;
return _sql;
}
/** Test-only escape hatch: force the next getRepositorySql call to open a fresh connection. */
export function resetRepositorySqlForTests(): void {
_sql = null;
_urlUsed = undefined;
}
/** Test-only: gracefully close the memoized connection (so a vitest run doesn't hang on an open socket) and clear it. */
export async function closeRepositorySqlForTests(): Promise<void> {
const sql = _sql;
_sql = null;
_urlUsed = undefined;
if (sql) await sql.end({ timeout: 5 });
}
@@ -0,0 +1,59 @@
/**
* src/repositories/in-memory-source-registry-repository.ts
* ============================================================
* The original SourceRegistryService storage (four Maps: rights,
* provenance, healthChecks, lastSuccess), extracted verbatim behind the
* SourceRegistryRepository seam. Health-check bounding (latest 100
* observations) is preserved exactly; `listHealthChecks` returns
* newest-first to match the Postgres implementation's `ORDER BY checked_at
* DESC` contract (the original Map-based `getHealth` read `list[list.length
* - 1]` for "most recent" and scanned backward for consecutive failures --
* callers now get the same data pre-ordered instead).
*/
import type { SourceHealthCheck, SourceProvenance, SourceRights } from "../types/source-registry";
import type { SourceRegistryRepository } from "./source-registry-repository";
export class InMemorySourceRegistryRepository implements SourceRegistryRepository {
private readonly rights = new Map<string, SourceRights>();
private readonly provenance = new Map<string, SourceProvenance>();
private readonly healthChecks = new Map<string, SourceHealthCheck[]>();
private readonly lastSuccess = new Map<string, string>();
async setRights(rights: SourceRights): Promise<void> {
this.rights.set(rights.source_id, rights);
}
async getRights(sourceId: string): Promise<SourceRights | undefined> {
return this.rights.get(sourceId);
}
async setProvenance(prov: SourceProvenance): Promise<void> {
this.provenance.set(prov.source_id, prov);
}
async getProvenance(sourceId: string): Promise<SourceProvenance | undefined> {
return this.provenance.get(sourceId);
}
async recordSuccess(sourceId: string, _tenantId: string, atIso: string): Promise<void> {
this.lastSuccess.set(sourceId, atIso);
}
async getLastSuccess(sourceId: string): Promise<string | undefined> {
return this.lastSuccess.get(sourceId);
}
async appendHealthCheck(check: SourceHealthCheck): Promise<void> {
const list = this.healthChecks.get(check.source_id) ?? [];
list.push(check);
// Bounded history -- health is an operational signal, not an audit log
// (AUD0 owns history); keep the latest 100 observations.
if (list.length > 100) list.splice(0, list.length - 100);
this.healthChecks.set(check.source_id, list);
}
async listHealthChecks(sourceId: string, limit = 100): Promise<SourceHealthCheck[]> {
const list = this.healthChecks.get(sourceId) ?? [];
return [...list].reverse().slice(0, limit);
}
}
@@ -0,0 +1,95 @@
/**
* src/repositories/in-memory-source-repository.ts
* ==================================================
* The original SourceService storage, extracted verbatim behind the
* SourceRepository seam (src/repositories/source-repository.ts) so
* PostgresSourceRepository can run the exact same business rules
* ("already exists", "name is required", "not found", filter semantics)
* against real Postgres instead. Used outside production when no
* DATABASE_URL is configured (resolveStoreMode), and directly by tests that
* want a hermetic, dependency-free SourceService.
*/
import type { CreateSourceInput, Source, SourceStatus, SourceType } from "../types/source";
import type { SourceListFilter, SourceRepository } from "./source-repository";
/** Storage shape: nullable columns, exactly mirroring `opt_sources`. */
interface SourceRecord {
source_id: string;
tenant_id: string;
project_id: string;
source_type: string;
name: string;
adapter_id: string | null;
access: string | null;
description: string | null;
status: string;
correlation_id: string | null;
created_at: string;
updated_at: string | null;
created_by: string | null;
}
export class InMemorySourceRepository implements SourceRepository {
private sources: Map<string, SourceRecord> = new Map();
async create(input: CreateSourceInput): Promise<Source> {
if (this.sources.has(input.source_id)) {
throw new Error(`Source ${input.source_id} already exists`);
}
if (!input.name) {
throw new Error("name is required");
}
const now = new Date().toISOString();
const record: SourceRecord = {
source_id: input.source_id,
tenant_id: input.tenant_id,
project_id: input.project_id,
source_type: input.source_type,
name: input.name,
adapter_id: input.adapter_id ?? null,
access: input.access ?? null,
description: input.description ?? null,
status: input.status ?? "active",
correlation_id: input.correlation_id ?? null,
created_at: now,
updated_at: null,
created_by: input.created_by ?? null,
};
this.sources.set(record.source_id, record);
return toSource(record);
}
async get(sourceId: string): Promise<Source> {
const record = this.sources.get(sourceId);
if (!record) throw new Error(`Source ${sourceId} not found`);
return toSource(record);
}
async list(filter?: SourceListFilter): Promise<Source[]> {
let results = Array.from(this.sources.values());
if (filter?.tenant_id) results = results.filter((r) => r.tenant_id === filter.tenant_id);
if (filter?.project_id) results = results.filter((r) => r.project_id === filter.project_id);
if (filter?.source_type) results = results.filter((r) => r.source_type === filter.source_type);
if (filter?.status) results = results.filter((r) => r.status === filter.status);
return results.map(toSource);
}
}
function toSource(r: SourceRecord): Source {
return {
source_id: r.source_id,
tenant_id: r.tenant_id,
project_id: r.project_id,
source_type: r.source_type as SourceType,
name: r.name,
adapter_id: r.adapter_id ?? undefined,
access: (r.access as Source["access"]) ?? undefined,
description: r.description ?? undefined,
status: r.status as SourceStatus,
correlation_id: r.correlation_id ?? undefined,
created_at: r.created_at,
updated_at: r.updated_at ?? undefined,
created_by: r.created_by ?? undefined,
};
}
@@ -0,0 +1,197 @@
/**
* src/repositories/postgres-source-registry-repository.ts
* ==========================================================
* Real Postgres backing for SourceRegistryRepository, against
* `opt_source_rights` / `opt_source_provenance` / `opt_source_health_checks`
* / `opt_source_freshness` (sql/002_source_registry.sql). Rights and
* provenance are one-row-per-source, upserted (matching the Map.set
* replace-in-place semantics they replace); health checks are append-only,
* bounded to the latest 100 rows per source_id after every insert; freshness
* is a single upserted "last success" row per source.
*/
import type {
SourceHealthCheck,
SourceHealthStatus,
SourceProvenance,
SourceRights,
} from "../types/source-registry";
import type { RepositorySql } from "./db";
import type { SourceRegistryRepository } from "./source-registry-repository";
function toIsoString(v: string | Date): string {
return v instanceof Date ? v.toISOString() : v;
}
interface RightsRow {
source_id: string;
tenant_id: string;
license_id: string;
license_url: string | null;
allowed_purposes: string[];
pii_allowed: boolean;
redistribution: string;
attribution: string | null;
verified_at: string;
verified_by: string;
notes: string | null;
}
function toRights(r: RightsRow): SourceRights {
return {
source_id: r.source_id,
tenant_id: r.tenant_id,
license_id: r.license_id,
license_url: r.license_url ?? undefined,
allowed_purposes: r.allowed_purposes ?? [],
pii_allowed: r.pii_allowed,
redistribution: r.redistribution as SourceRights["redistribution"],
attribution: r.attribution ?? undefined,
verified_at: r.verified_at,
verified_by: r.verified_by,
notes: r.notes ?? undefined,
};
}
interface ProvenanceRow {
source_id: string;
tenant_id: string;
method: string;
authority: string | null;
compliance_evidence_ref: string | null;
cadence: string | null;
freshness_sla_hours: string | number | null;
updated_at: string | Date;
}
function toProvenance(r: ProvenanceRow): SourceProvenance {
return {
source_id: r.source_id,
tenant_id: r.tenant_id,
method: r.method as SourceProvenance["method"],
authority: r.authority ?? undefined,
compliance_evidence_ref: r.compliance_evidence_ref ?? undefined,
cadence: r.cadence ?? undefined,
freshness_sla_hours: r.freshness_sla_hours === null ? undefined : Number(r.freshness_sla_hours),
updated_at: toIsoString(r.updated_at),
};
}
interface HealthCheckRow {
source_id: string;
tenant_id: string;
status: string;
checked_at: string | Date;
latency_ms: number | null;
detail: string | null;
}
function toHealthCheck(r: HealthCheckRow): SourceHealthCheck {
return {
source_id: r.source_id,
tenant_id: r.tenant_id,
status: r.status as SourceHealthStatus,
checked_at: toIsoString(r.checked_at),
latency_ms: r.latency_ms ?? undefined,
detail: r.detail ?? undefined,
};
}
export class PostgresSourceRegistryRepository implements SourceRegistryRepository {
constructor(private readonly sql: RepositorySql) {}
async setRights(rights: SourceRights): Promise<void> {
await this.sql`
INSERT INTO opt_source_rights
(source_id, tenant_id, license_id, license_url, allowed_purposes, pii_allowed, redistribution, attribution, verified_at, verified_by, notes, updated_at)
VALUES
(${rights.source_id}, ${rights.tenant_id}, ${rights.license_id}, ${rights.license_url ?? null},
${this.sql.array(rights.allowed_purposes)}, ${rights.pii_allowed}, ${rights.redistribution},
${rights.attribution ?? null}, ${rights.verified_at}, ${rights.verified_by}, ${rights.notes ?? null}, now())
ON CONFLICT (source_id) DO UPDATE SET
tenant_id = EXCLUDED.tenant_id,
license_id = EXCLUDED.license_id,
license_url = EXCLUDED.license_url,
allowed_purposes = EXCLUDED.allowed_purposes,
pii_allowed = EXCLUDED.pii_allowed,
redistribution = EXCLUDED.redistribution,
attribution = EXCLUDED.attribution,
verified_at = EXCLUDED.verified_at,
verified_by = EXCLUDED.verified_by,
notes = EXCLUDED.notes,
updated_at = now()`;
}
async getRights(sourceId: string): Promise<SourceRights | undefined> {
const rows = await this.sql<RightsRow[]>`
SELECT source_id, tenant_id, license_id, license_url, allowed_purposes, pii_allowed, redistribution, attribution, verified_at, verified_by, notes
FROM opt_source_rights WHERE source_id = ${sourceId} LIMIT 1`;
return rows[0] ? toRights(rows[0]) : undefined;
}
async setProvenance(prov: SourceProvenance): Promise<void> {
await this.sql`
INSERT INTO opt_source_provenance
(source_id, tenant_id, method, authority, compliance_evidence_ref, cadence, freshness_sla_hours, updated_at)
VALUES
(${prov.source_id}, ${prov.tenant_id}, ${prov.method}, ${prov.authority ?? null},
${prov.compliance_evidence_ref ?? null}, ${prov.cadence ?? null}, ${prov.freshness_sla_hours ?? null}, ${prov.updated_at})
ON CONFLICT (source_id) DO UPDATE SET
tenant_id = EXCLUDED.tenant_id,
method = EXCLUDED.method,
authority = EXCLUDED.authority,
compliance_evidence_ref = EXCLUDED.compliance_evidence_ref,
cadence = EXCLUDED.cadence,
freshness_sla_hours = EXCLUDED.freshness_sla_hours,
updated_at = EXCLUDED.updated_at`;
}
async getProvenance(sourceId: string): Promise<SourceProvenance | undefined> {
const rows = await this.sql<ProvenanceRow[]>`
SELECT source_id, tenant_id, method, authority, compliance_evidence_ref, cadence, freshness_sla_hours, updated_at
FROM opt_source_provenance WHERE source_id = ${sourceId} LIMIT 1`;
return rows[0] ? toProvenance(rows[0]) : undefined;
}
async recordSuccess(sourceId: string, tenantId: string, atIso: string): Promise<void> {
await this.sql`
INSERT INTO opt_source_freshness (source_id, tenant_id, last_success_at)
VALUES (${sourceId}, ${tenantId}, ${atIso})
ON CONFLICT (source_id) DO UPDATE SET
last_success_at = EXCLUDED.last_success_at,
tenant_id = EXCLUDED.tenant_id`;
}
async getLastSuccess(sourceId: string): Promise<string | undefined> {
const rows = await this.sql<{ last_success_at: string | Date }[]>`
SELECT last_success_at FROM opt_source_freshness WHERE source_id = ${sourceId} LIMIT 1`;
return rows[0] ? toIsoString(rows[0].last_success_at) : undefined;
}
async appendHealthCheck(check: SourceHealthCheck): Promise<void> {
await this.sql`
INSERT INTO opt_source_health_checks (source_id, tenant_id, status, checked_at, latency_ms, detail)
VALUES (${check.source_id}, ${check.tenant_id}, ${check.status}, ${check.checked_at}, ${check.latency_ms ?? null}, ${check.detail ?? null})`;
// Bounded history -- keep only the latest 100 observations per source
// (health is an operational signal, not an audit log; AUD0 owns history),
// same bound InMemorySourceRegistryRepository enforces on write.
await this.sql`
DELETE FROM opt_source_health_checks
WHERE source_id = ${check.source_id}
AND health_check_id NOT IN (
SELECT health_check_id FROM opt_source_health_checks
WHERE source_id = ${check.source_id}
ORDER BY checked_at DESC, health_check_id DESC
LIMIT 100
)`;
}
async listHealthChecks(sourceId: string, limit = 100): Promise<SourceHealthCheck[]> {
const rows = await this.sql<HealthCheckRow[]>`
SELECT source_id, tenant_id, status, checked_at, latency_ms, detail
FROM opt_source_health_checks
WHERE source_id = ${sourceId}
ORDER BY checked_at DESC, health_check_id DESC
LIMIT ${limit}`;
return rows.map(toHealthCheck);
}
}
@@ -0,0 +1,98 @@
/**
* src/repositories/postgres-source-repository.ts
* =================================================
* Real Postgres backing for SourceRepository, against `opt_sources`
* (sql/002_source_registry.sql). Same validation/error semantics as
* InMemorySourceRepository ("already exists", "name is required", "not
* found" -- exact message text preserved so every existing caller/test that
* matches on those strings keeps working unmodified).
*/
import type { CreateSourceInput, Source, SourceStatus, SourceType } from "../types/source";
import type { RepositorySql } from "./db";
import type { SourceListFilter, SourceRepository } from "./source-repository";
interface SourceRow {
source_id: string;
tenant_id: string;
project_id: string;
source_type: string;
name: string;
adapter_id: string | null;
access: string | null;
description: string | null;
status: string;
correlation_id: string | null;
created_at: string | Date;
updated_at: string | Date | null;
created_by: string | null;
}
function toIsoString(v: string | Date): string {
return v instanceof Date ? v.toISOString() : v;
}
function toSource(r: SourceRow): Source {
return {
source_id: r.source_id,
tenant_id: r.tenant_id,
project_id: r.project_id,
source_type: r.source_type as SourceType,
name: r.name,
adapter_id: r.adapter_id ?? undefined,
access: (r.access as Source["access"]) ?? undefined,
description: r.description ?? undefined,
status: r.status as SourceStatus,
correlation_id: r.correlation_id ?? undefined,
created_at: toIsoString(r.created_at),
updated_at: r.updated_at ? toIsoString(r.updated_at) : undefined,
created_by: r.created_by ?? undefined,
};
}
export class PostgresSourceRepository implements SourceRepository {
constructor(private readonly sql: RepositorySql) {}
async create(input: CreateSourceInput): Promise<Source> {
if (!input.name) throw new Error("name is required");
const existing = await this.sql<{ source_id: string }[]>`
SELECT source_id FROM opt_sources WHERE source_id = ${input.source_id} LIMIT 1`;
if (existing.length > 0) {
throw new Error(`Source ${input.source_id} already exists`);
}
const rows = await this.sql<SourceRow[]>`
INSERT INTO opt_sources
(source_id, tenant_id, project_id, source_type, name, adapter_id, access, description, status, correlation_id, created_by)
VALUES
(${input.source_id}, ${input.tenant_id}, ${input.project_id}, ${input.source_type}, ${input.name},
${input.adapter_id ?? null}, ${input.access ?? null}, ${input.description ?? null},
${input.status ?? "active"}, ${input.correlation_id ?? null}, ${input.created_by ?? null})
RETURNING source_id, tenant_id, project_id, source_type, name, adapter_id, access, description,
status, correlation_id, created_at, updated_at, created_by`;
return toSource(rows[0]);
}
async get(sourceId: string): Promise<Source> {
const rows = await this.sql<SourceRow[]>`
SELECT source_id, tenant_id, project_id, source_type, name, adapter_id, access, description,
status, correlation_id, created_at, updated_at, created_by
FROM opt_sources WHERE source_id = ${sourceId} LIMIT 1`;
if (!rows[0]) throw new Error(`Source ${sourceId} not found`);
return toSource(rows[0]);
}
async list(filter?: SourceListFilter): Promise<Source[]> {
const rows = await this.sql<SourceRow[]>`
SELECT source_id, tenant_id, project_id, source_type, name, adapter_id, access, description,
status, correlation_id, created_at, updated_at, created_by
FROM opt_sources
WHERE 1=1
${filter?.tenant_id ? this.sql`AND tenant_id = ${filter.tenant_id}` : this.sql``}
${filter?.project_id ? this.sql`AND project_id = ${filter.project_id}` : this.sql``}
${filter?.source_type ? this.sql`AND source_type = ${filter.source_type}` : this.sql``}
${filter?.status ? this.sql`AND status = ${filter.status}` : this.sql``}
ORDER BY created_at ASC`;
return rows.map(toSource);
}
}
@@ -0,0 +1,45 @@
/**
* src/repositories/source-registry-repository.ts
* =================================================
* Storage seam for the four Maps src/services/source-registry-service.ts
* used to hold rights, provenance, health checks, and last-success
* timestamps. Same split as source-repository.ts: one interface, an
* in-memory implementation (extracted verbatim) and a Postgres
* implementation (sql/002_source_registry.sql's `opt_source_rights`,
* `opt_source_provenance`, `opt_source_health_checks`, `opt_source_freshness`
* tables), chosen once via `resolveStoreMode`.
*
* Tenant isolation is NOT this layer's job -- exactly as before, when these
* were bare Maps keyed only by source_id. SourceRegistryService's
* `assertSourceTenant` gate is what enforces the cross-tenant NOT_FOUND
* posture (§50), and it runs before every call into this repository. This
* layer trusts the caller, same as the Maps did.
*/
import { resolveStoreMode } from "../platform/dependency-mode";
import type { SourceHealthCheck, SourceProvenance, SourceRights } from "../types/source-registry";
import { getRepositorySql, repositoryDatabaseUrl } from "./db";
import { InMemorySourceRegistryRepository } from "./in-memory-source-registry-repository";
import { PostgresSourceRegistryRepository } from "./postgres-source-registry-repository";
export interface SourceRegistryRepository {
setRights(rights: SourceRights): Promise<void>;
getRights(sourceId: string): Promise<SourceRights | undefined>;
setProvenance(prov: SourceProvenance): Promise<void>;
getProvenance(sourceId: string): Promise<SourceProvenance | undefined>;
recordSuccess(sourceId: string, tenantId: string, atIso: string): Promise<void>;
getLastSuccess(sourceId: string): Promise<string | undefined>;
appendHealthCheck(check: SourceHealthCheck): Promise<void>;
/** Newest-first, bounded to the latest `limit` observations (default 100) -- health is an operational signal, not an audit log; AUD0 owns history. */
listHealthChecks(sourceId: string, limit?: number): Promise<SourceHealthCheck[]>;
}
export function createSourceRegistryRepository(databaseUrl?: string): SourceRegistryRepository {
const mode = resolveStoreMode("0pt.source_registry.metadata", databaseUrl ?? repositoryDatabaseUrl());
if (mode.mode === "postgres") {
return new PostgresSourceRegistryRepository(getRepositorySql(mode.url));
}
return new InMemorySourceRegistryRepository();
}
+44
View File
@@ -0,0 +1,44 @@
/**
* src/repositories/source-repository.ts
* =======================================
* Storage seam for src/services/source-service.ts (0pt.source.schema.v1).
* Extracted from what used to be a bare `new Map()` inside SourceService
* itself so the same business logic (validation, "already exists"/"not
* found" semantics, filtering) can run against either an in-memory Map
* (tests, local dev, non-production with no DATABASE_URL) or real Postgres
* (sql/002_source_registry.sql's `opt_sources` table) without the service
* layer knowing which.
*
* `createSourceRepository` is the single choice point, delegating to
* `resolveStoreMode` (src/platform/dependency-mode.ts) -- same
* production-safety contract as every other P0 store: no DATABASE_URL in a
* production runtime throws DEPENDENCY_UNAVAILABLE at construction rather
* than silently serving the in-memory fallback.
*/
import { resolveStoreMode } from "../platform/dependency-mode";
import type { CreateSourceInput, Source, SourceStatus, SourceType } from "../types/source";
import { getRepositorySql, repositoryDatabaseUrl } from "./db";
import { InMemorySourceRepository } from "./in-memory-source-repository";
import { PostgresSourceRepository } from "./postgres-source-repository";
export interface SourceListFilter {
tenant_id?: string;
project_id?: string;
source_type?: SourceType;
status?: SourceStatus;
}
export interface SourceRepository {
create(input: CreateSourceInput): Promise<Source>;
/** Throws (message contains "not found") when no such source exists -- matches the pre-migration Map behavior every caller already handles. */
get(sourceId: string): Promise<Source>;
list(filter?: SourceListFilter): Promise<Source[]>;
}
export function createSourceRepository(databaseUrl?: string): SourceRepository {
const mode = resolveStoreMode("0pt.source_registry.sources", databaseUrl ?? repositoryDatabaseUrl());
if (mode.mode === "postgres") {
return new PostgresSourceRepository(getRepositorySql(mode.url));
}
return new InMemorySourceRepository();
}
+6 -2
View File
@@ -1,4 +1,8 @@
import type { SourceRatePolicy, SourceRatePolicyStatus, CreateSourceRatePolicyInput } from "../types/source-rate-policy";
import type {
SourceRatePolicy,
SourceRatePolicyStatus,
CreateSourceRatePolicyInput,
} from "../types/source-rate-policy";
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import { SourceService } from "./source-service";
@@ -33,7 +37,7 @@ export class SourceRatePolicyService {
constructor(opts?: { aud0BaseUrl?: string; sourceService?: SourceService }) {
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.sourceService = opts?.sourceService ?? new SourceService(opts?.aud0BaseUrl);
this.sourceService = opts?.sourceService ?? new SourceService({ aud0BaseUrl: opts?.aud0BaseUrl });
}
async createSourceRatePolicy(input: CreateSourceRatePolicyInput): Promise<SourceRatePolicy> {
+79 -48
View File
@@ -14,20 +14,33 @@
* Tenant isolation: every read path REQUIRES a tenant_id and filters on it —
* a source registered by tenant A is invisible to tenant B (tested in
* tests/p0/source-registry.test.ts, incl. the cross-tenant attack case §50).
* `getHealth`/`getFreshness` now take tenant_id directly and run the same
* `assertSourceTenant` gate every other accessor does — they are exposed as
* standalone HTTP routes now (app/api/0pt/sources/[sourceId]/health,
* .../freshness), so they can no longer rely on an earlier caller having
* already checked tenancy on their behalf.
*
* Purpose enforcement: `assertPurposeAllowed` is the single hook other 0pt
* capabilities call before using a source for a given purpose (SearchService
* and DatasetService both do). No rights record ⇒ fail closed (deny).
*
* Persistence: in-memory Maps behind the same "pure core, thin registry"
* shape as the sibling P0 services. Production boot is protected by
* `resolveStoreMode` in the composition root (app/api routes construct with
* the shared registry): with no database configured a production runtime
* throws DEPENDENCY_UNAVAILABLE instead of serving Maps (Master-DoD §3).
* Persistence: rights/provenance/health/freshness now live behind
* SourceRegistryRepository (src/repositories/source-registry-repository.ts)
* — real Postgres (`opt_source_rights`/`opt_source_provenance`/
* `opt_source_health_checks`/`opt_source_freshness`,
* sql/002_source_registry.sql) when DATABASE_URL is configured, in-memory
* Maps otherwise. `resolveStoreMode` still fails production boot hard with
* no DATABASE_URL (Master-DoD §3) — the difference from before is that a
* configured production now actually WORKS (opt_sources/etc. exist), not
* just fails safely.
*/
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import { SiaxError } from "../platform/errors";
import { SourceService } from "./source-service";
import {
createSourceRegistryRepository,
type SourceRegistryRepository,
} from "../repositories/source-registry-repository";
import { SourceService, type SourceServiceOptions } from "./source-service";
import { SourceRatePolicyService } from "./source-rate-policy-service";
import type { CreateSourceInput, Source } from "../types/source";
import type {
@@ -46,21 +59,27 @@ export interface RegisterSourceInput extends CreateSourceInput {
}
export class SourceRegistryService {
private readonly rights = new Map<string, SourceRights>();
private readonly provenance = new Map<string, SourceProvenance>();
private readonly healthChecks = new Map<string, SourceHealthCheck[]>();
private readonly lastSuccess = new Map<string, string>();
private readonly repo: SourceRegistryRepository;
private readonly aud0: Aud0Client;
readonly sources: SourceService;
readonly ratePolicies: SourceRatePolicyService;
constructor(opts?: {
aud0BaseUrl?: string;
/** Explicit DATABASE_URL override — mainly for tests. Falls back to resolveStoreMode's own env lookup when omitted. */
databaseUrl?: string;
sourceService?: SourceService;
ratePolicyService?: SourceRatePolicyService;
/** Inject a repository directly (tests) — bypasses resolveStoreMode entirely. */
repository?: SourceRegistryRepository;
}) {
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.sources = opts?.sourceService ?? new SourceService(opts?.aud0BaseUrl);
this.repo = opts?.repository ?? createSourceRegistryRepository(opts?.databaseUrl);
const sourceServiceOpts: SourceServiceOptions = {
aud0BaseUrl: opts?.aud0BaseUrl,
databaseUrl: opts?.databaseUrl,
};
this.sources = opts?.sourceService ?? new SourceService(sourceServiceOpts);
this.ratePolicies =
opts?.ratePolicyService ??
new SourceRatePolicyService({ aud0BaseUrl: opts?.aud0BaseUrl, sourceService: this.sources });
@@ -91,7 +110,7 @@ export class SourceRegistryService {
// -------------------------------------------------------------------------
async setRights(rights: SourceRights): Promise<SourceRights> {
await this.assertSourceTenant(rights.source_id, rights.tenant_id);
this.rights.set(rights.source_id, rights);
await this.repo.setRights(rights);
await this.emit("0pt.source_rights_set", rights.tenant_id, rights.source_id, {
license_id: rights.license_id,
allowed_purposes: rights.allowed_purposes,
@@ -102,7 +121,7 @@ export class SourceRegistryService {
async getRights(sourceId: string, tenantId: string): Promise<SourceRights | undefined> {
await this.assertSourceTenant(sourceId, tenantId);
return this.rights.get(sourceId);
return this.repo.getRights(sourceId);
}
/**
@@ -112,19 +131,19 @@ export class SourceRegistryService {
*/
async assertPurposeAllowed(sourceId: string, tenantId: string, purpose: string): Promise<SourceRights> {
await this.assertSourceTenant(sourceId, tenantId);
const rights = this.rights.get(sourceId);
const rights = await this.repo.getRights(sourceId);
if (!rights) {
throw new SiaxError(
"AUTHORIZATION_DENIED",
"0pt.source_registry",
`source ${sourceId} has no rights record — purpose "${purpose}" denied (fail closed)`
`source ${sourceId} has no rights record — purpose "${purpose}" denied (fail closed)`,
);
}
if (!rights.allowed_purposes.includes(purpose)) {
throw new SiaxError(
"AUTHORIZATION_DENIED",
"0pt.source_registry",
`purpose "${purpose}" is not in source ${sourceId}'s allowed purposes`
`purpose "${purpose}" is not in source ${sourceId}'s allowed purposes`,
);
}
return rights;
@@ -135,7 +154,7 @@ export class SourceRegistryService {
// -------------------------------------------------------------------------
async setProvenance(prov: SourceProvenance): Promise<SourceProvenance> {
await this.assertSourceTenant(prov.source_id, prov.tenant_id);
this.provenance.set(prov.source_id, prov);
await this.repo.setProvenance(prov);
await this.emit("0pt.source_provenance_set", prov.tenant_id, prov.source_id, {
method: prov.method,
authority: prov.authority,
@@ -146,18 +165,19 @@ export class SourceRegistryService {
async getProvenance(sourceId: string, tenantId: string): Promise<SourceProvenance | undefined> {
await this.assertSourceTenant(sourceId, tenantId);
return this.provenance.get(sourceId);
return this.repo.getProvenance(sourceId);
}
/** Record a successful data observation (drives freshness). */
async recordSuccess(sourceId: string, tenantId: string, atIso?: string): Promise<void> {
await this.assertSourceTenant(sourceId, tenantId);
this.lastSuccess.set(sourceId, atIso ?? new Date().toISOString());
await this.repo.recordSuccess(sourceId, tenantId, atIso ?? new Date().toISOString());
}
getFreshness(sourceId: string, now: Date = new Date()): SourceFreshness {
const prov = this.provenance.get(sourceId);
const last = this.lastSuccess.get(sourceId);
async getFreshness(sourceId: string, tenantId: string, now: Date = new Date()): Promise<SourceFreshness> {
await this.assertSourceTenant(sourceId, tenantId);
const prov = await this.repo.getProvenance(sourceId);
const last = await this.repo.getLastSuccess(sourceId);
const sla = prov?.freshness_sla_hours;
if (!last) {
return { source_id: sourceId, freshness_sla_hours: sla, fresh: sla === undefined };
@@ -177,24 +197,21 @@ export class SourceRegistryService {
// -------------------------------------------------------------------------
async recordHealthCheck(check: SourceHealthCheck): Promise<SourceHealth> {
await this.assertSourceTenant(check.source_id, check.tenant_id);
const list = this.healthChecks.get(check.source_id) ?? [];
list.push(check);
// Bounded history — health is an operational signal, not an audit log
// (AUD0 owns history); keep the latest 100 observations.
if (list.length > 100) list.splice(0, list.length - 100);
this.healthChecks.set(check.source_id, list);
await this.repo.appendHealthCheck(check);
if (check.status === "up") {
await this.recordSuccess(check.source_id, check.tenant_id, check.checked_at);
}
return this.getHealth(check.source_id);
return this.getHealth(check.source_id, check.tenant_id);
}
getHealth(sourceId: string): SourceHealth {
const list = this.healthChecks.get(sourceId) ?? [];
const last = list[list.length - 1];
async getHealth(sourceId: string, tenantId: string): Promise<SourceHealth> {
await this.assertSourceTenant(sourceId, tenantId);
// Newest-first (repository contract) — index 0 is the most recent check.
const list = await this.repo.listHealthChecks(sourceId);
const last = list[0];
let consecutiveFailures = 0;
for (let i = list.length - 1; i >= 0; i--) {
if (list[i].status === "down" || list[i].status === "degraded") consecutiveFailures++;
for (const check of list) {
if (check.status === "down" || check.status === "degraded") consecutiveFailures++;
else break;
}
const status: SourceHealthStatus = last?.status ?? "unknown";
@@ -223,27 +240,41 @@ export class SourceRegistryService {
source_type: filter.source_type,
status: filter.status,
});
if (filter.purpose !== undefined) {
results = results.filter((s) => {
const rights = this.rights.get(s.source_id);
return rights !== undefined && rights.allowed_purposes.includes(filter.purpose as string);
});
}
if (filter.redistribution !== undefined) {
results = results.filter((s) => this.rights.get(s.source_id)?.redistribution === filter.redistribution);
if (filter.purpose !== undefined || filter.redistribution !== undefined) {
const rightsEntries = await Promise.all(
results.map(async (s) => [s.source_id, await this.repo.getRights(s.source_id)] as const),
);
const rightsBySource = new Map(rightsEntries);
if (filter.purpose !== undefined) {
results = results.filter((s) => {
const rights = rightsBySource.get(s.source_id);
return rights !== undefined && rights.allowed_purposes.includes(filter.purpose as string);
});
}
if (filter.redistribution !== undefined) {
results = results.filter(
(s) => rightsBySource.get(s.source_id)?.redistribution === filter.redistribution,
);
}
}
return results;
}
async describeSource(sourceId: string, tenantId: string): Promise<SourceDescriptor> {
const source = await this.assertSourceTenant(sourceId, tenantId);
const policies = await this.ratePolicies.listSourceRatePolicies({ tenant_id: tenantId, source_id: sourceId });
const [policies, rights, provenance, health, freshness] = await Promise.all([
this.ratePolicies.listSourceRatePolicies({ tenant_id: tenantId, source_id: sourceId }),
this.repo.getRights(sourceId),
this.repo.getProvenance(sourceId),
this.getHealth(sourceId, tenantId),
this.getFreshness(sourceId, tenantId),
]);
return {
source,
rights: this.rights.get(sourceId),
provenance: this.provenance.get(sourceId),
health: this.getHealth(sourceId),
freshness: this.getFreshness(sourceId),
rights,
provenance,
health,
freshness,
rate_policy: policies[0],
};
}
@@ -276,7 +307,7 @@ export class SourceRegistryService {
eventType: string,
tenantId: string,
sourceId: string,
payload: Record<string, unknown>
payload: Record<string, unknown>,
): Promise<void> {
await this.aud0.ingestEvent({
event_id: `evt_${eventType}_${sourceId}_${Date.now()}`,
+30
View File
@@ -0,0 +1,30 @@
/**
* src/services/source-registry.ts
* =================================
* Shared singleton instances for the source-registry route group
* (app/api/0pt/sources/*), matching src/services/lakehouse-mapping-registry.ts's
* precedent.
*
* BEFORE_STATE.md §1 flagged this as materially worse than plain
* in-memory-per-module: each route file previously constructed its own
* `new SourceService()` (and `new SourceRatePolicyService()`), so a source
* registered via one route module was invisible to another within the same
* running process — "not even a shared process-wide singleton". Once
* DATABASE_URL is configured this stops mattering (state lives in Postgres,
* genuinely shared across processes/deploys — see src/repositories/*), but
* it still matters for the in-memory (no DATABASE_URL) dev/test mode, where
* state must be shared within one process to be usable at all. Constructing
* all three here once, wired together, and importing them everywhere fixes
* both call sites this repo already had (sources/rate-policies/*,
* sources/[sourceId]/query) plus every new route this pass adds.
*/
import { SourceService } from "./source-service";
import { SourceRatePolicyService } from "./source-rate-policy-service";
import { SourceRegistryService } from "./source-registry-service";
export const sourceService = new SourceService();
export const sourceRatePolicyService = new SourceRatePolicyService({ sourceService });
export const sourceRegistryService = new SourceRegistryService({
sourceService,
ratePolicyService: sourceRatePolicyService,
});
+43 -99
View File
@@ -7,107 +7,74 @@
* and `SourceRatePolicyService` (src/services/source-rate-policy-service.ts)
* both look a `source_id` up against — see docs/P0_QUERY_SOURCE_FLOW.md.
*
* Deliberately small and in-memory, same "pure core, thin registry" shape
* every other P0 service in this repo already uses; a real deployment backs
* this with `opt_sources`. In-memory scope: each `new SourceService()` owns
* its own registry — the pre-existing limitation docs/P0_QUERY_SOURCE_FLOW.md's
* "Known gaps" section documents (a Source registered via one API route module
* is not yet visible to another route module in the same running process).
* Callers that need a shared view pass an existing instance in via the
* `sourceService` constructor option, exactly like SourceQueryService and
* SourceRatePolicyService already do.
* Persistence: storage now lives behind SourceRepository
* (src/repositories/source-repository.ts) — real Postgres (`opt_sources`,
* sql/002_source_registry.sql) when DATABASE_URL is configured, an
* in-memory Map otherwise (non-production only; resolveStoreMode fails hard
* in production with no URL). This service still owns the business rules
* (validation, tenant-agnostic CRUD orchestration) and the AUD0 emission —
* only the storage moved.
*
* Registering a source emits a best-effort AUD0 `0pt.source_created` event —
* the same "no mutation without an evented trail" doctrine every other P0
* registry service follows. With no AUD0_URL configured that emission is a
* no-op (see aud0-client.ts), so this stays hermetic in tests and local dev.
* Callers that need a shared view across route modules should use the
* shared singleton in src/services/source-registry.ts rather than
* constructing a fresh `new SourceService()` per module — see that file's
* doc comment for the "each route owned its own registry" history this
* fixes.
*/
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import {
createSourceRepository,
type SourceRepository,
type SourceListFilter,
} from "../repositories/source-repository";
import type { CreateSourceInput, Source, SourceStatus, SourceType } from "../types/source";
/**
* Storage shape: nullable columns, exactly as the eventual `opt_sources` table
* would hold them. `toSource()` converts back to the `Source` DTO (undefined
* rather than null) so callers never have to handle both.
*/
interface SourceRecord {
source_id: string;
tenant_id: string;
project_id: string;
source_type: string;
name: string;
adapter_id: string | null;
access: string | null;
description: string | null;
status: string;
correlation_id: string | null;
created_at: string;
updated_at: string | null;
created_by: string | null;
export interface SourceServiceOptions {
aud0BaseUrl?: string;
/** Explicit DATABASE_URL override — mainly for tests pointing at a specific Postgres instance. Falls back to resolveStoreMode's own env lookup when omitted. */
databaseUrl?: string;
/** Inject a repository directly (tests) — bypasses resolveStoreMode entirely. */
repository?: SourceRepository;
}
export class SourceService {
private sources: Map<string, SourceRecord> = new Map();
private aud0: Aud0Client;
private readonly repo: SourceRepository;
private readonly aud0: Aud0Client;
constructor(aud0BaseUrl?: string) {
this.aud0 = createAud0Client(aud0BaseUrl);
constructor(opts?: SourceServiceOptions) {
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.repo = opts?.repository ?? createSourceRepository(opts?.databaseUrl);
}
async createSource(input: CreateSourceInput): Promise<Source> {
if (this.sources.has(input.source_id)) {
throw new Error(`Source ${input.source_id} already exists`);
}
if (!input.name) {
throw new Error("name is required");
}
const now = new Date().toISOString();
const record: SourceRecord = {
source_id: input.source_id,
tenant_id: input.tenant_id,
project_id: input.project_id,
source_type: input.source_type,
name: input.name,
adapter_id: input.adapter_id ?? null,
access: input.access ?? null,
description: input.description ?? null,
status: input.status ?? "active",
correlation_id: input.correlation_id ?? null,
created_at: now,
updated_at: null,
created_by: input.created_by ?? null,
};
this.sources.set(record.source_id, record);
const source = await this.repo.create(input);
await this.aud0.ingestEvent({
event_id: `evt_0pt_source_created_${record.source_id}`,
event_id: `evt_0pt_source_created_${source.source_id}`,
event_type: "0pt.source_created",
tenant_id: record.tenant_id,
project_id: record.project_id,
actor_id: record.created_by ?? "system",
tenant_id: source.tenant_id,
project_id: source.project_id,
actor_id: source.created_by ?? "system",
app_id: "0pt",
capability_id: "0pt.create_source",
resource_type: "source",
resource_id: record.source_id,
resource_id: source.source_id,
risk_level: "L2",
payload: {
source_type: record.source_type,
name: record.name,
adapter_id: record.adapter_id,
access: record.access,
source_type: source.source_type,
name: source.name,
adapter_id: source.adapter_id,
access: source.access,
},
created_at: now,
correlation_id: record.correlation_id ?? undefined,
created_at: source.created_at,
correlation_id: source.correlation_id,
});
return this.toSource(record);
return source;
}
async getSource(sourceId: string): Promise<Source> {
const record = this.sources.get(sourceId);
if (!record) throw new Error(`Source ${sourceId} not found`);
return this.toSource(record);
return this.repo.get(sourceId);
}
async listSources(filter?: {
@@ -116,29 +83,6 @@ export class SourceService {
source_type?: SourceType;
status?: SourceStatus;
}): Promise<Source[]> {
let results = Array.from(this.sources.values());
if (filter?.tenant_id) results = results.filter((r) => r.tenant_id === filter.tenant_id);
if (filter?.project_id) results = results.filter((r) => r.project_id === filter.project_id);
if (filter?.source_type) results = results.filter((r) => r.source_type === filter.source_type);
if (filter?.status) results = results.filter((r) => r.status === filter.status);
return results.map((r) => this.toSource(r));
}
private toSource(r: SourceRecord): Source {
return {
source_id: r.source_id,
tenant_id: r.tenant_id,
project_id: r.project_id,
source_type: r.source_type as SourceType,
name: r.name,
adapter_id: r.adapter_id ?? undefined,
access: (r.access as Source["access"]) ?? undefined,
description: r.description ?? undefined,
status: r.status as SourceStatus,
correlation_id: r.correlation_id ?? undefined,
created_at: r.created_at,
updated_at: r.updated_at ?? undefined,
created_by: r.created_by ?? undefined,
};
return this.repo.list(filter as SourceListFilter | undefined);
}
}
+58
View File
@@ -95,6 +95,64 @@ export const createSourceRatePolicySchema = z.object({
created_by: z.string().min(1).optional(),
});
/** Nested `rights` block on POST /api/0pt/sources, and the body of PUT /api/0pt/sources/[sourceId]/rights -- mirrors SourceRights (minus source_id/tenant_id, which come from the URL/outer body). */
const sourceRightsInputSchema = z.object({
license_id: z.string().min(1),
license_url: z.string().min(1).optional(),
allowed_purposes: z.array(z.string().min(1)).default([]),
pii_allowed: z.boolean(),
redistribution: z.enum(['internal', 'customer_deliverable', 'public', 'prohibited']),
attribution: z.string().min(1).optional(),
verified_at: z.string().min(1),
verified_by: z.string().min(1),
notes: z.string().min(1).optional(),
});
/** Nested `provenance` block on POST /api/0pt/sources, and the body of PUT /api/0pt/sources/[sourceId]/provenance -- mirrors SourceProvenance (minus source_id/tenant_id/updated_at, which are set by the service). */
const sourceProvenanceInputSchema = z.object({
method: z.enum(['official_api', 'bulk_download', 'authorized_web', 'licensed_feed', 'manual_upload']),
authority: z.string().min(1).optional(),
compliance_evidence_ref: z.string().min(1).optional(),
cadence: z.string().min(1).optional(),
freshness_sla_hours: z.number().nonnegative().optional(),
});
/** POST /api/0pt/sources -- source.register. */
export const registerSourceSchema = z.object({
source_id: z.string().min(1),
tenant_id: z.string().min(1),
project_id: z.string().min(1),
source_type: z.enum(['database', 'file', 'web', 'api', 'stream', 'manual']),
name: z.string().min(1),
adapter_id: z.string().min(1).optional(),
access: z.enum(['api', 'bulk', 'authorized_web', 'licensed']).optional(),
description: z.string().min(1).optional(),
status: z.enum(['active', 'paused', 'archived', 'deprecated']).optional(),
correlation_id: z.string().min(1).optional(),
created_by: z.string().min(1).optional(),
rights: sourceRightsInputSchema.optional(),
provenance: sourceProvenanceInputSchema.optional(),
});
/** PUT /api/0pt/sources/[sourceId]/rights -- source.set_rights. */
export const setSourceRightsSchema = sourceRightsInputSchema.extend({
tenant_id: z.string().min(1),
});
/** PUT /api/0pt/sources/[sourceId]/provenance -- source.set_provenance. */
export const setSourceProvenanceSchema = sourceProvenanceInputSchema.extend({
tenant_id: z.string().min(1),
});
/** POST /api/0pt/sources/[sourceId]/health -- source.health (record a health check). */
export const recordSourceHealthCheckSchema = z.object({
tenant_id: z.string().min(1),
status: z.enum(['up', 'degraded', 'down', 'unknown']),
checked_at: z.string().min(1).optional(),
latency_ms: z.number().nonnegative().optional(),
detail: z.string().min(1).optional(),
});
/** POST /api/0pt/lakehouse/mappings -- mirrors MapDatasetToIcebergTableInput. */
export const mapDatasetToIcebergTableSchema = z.object({
tenant_id: z.string().min(1),
@@ -0,0 +1,340 @@
/**
* REAL Postgres integration test (Master-DoD §48/§50) for the Source
* Registry's new persistence layer (src/repositories/*,
* sql/002_source_registry.sql) -- the P010 task's central deliverable: this
* is what proves the Map -> Postgres migration actually works, not just
* that it typechecks.
*
* Same honest-degrade convention as tests/p0/iceberg-write-service.realinfra.test.ts
* and tests/integration/analytics-clickhouse.integration.test.ts: gated on
* an env var (SOURCE_REGISTRY_TEST_DATABASE_URL), SKIPS HONESTLY when unset,
* never fabricates success. Point it at a disposable Postgres, e.g.:
*
* docker run -d --name opt-source-registry-pgtest \
* -e POSTGRES_USER=opt -e POSTGRES_PASSWORD=opt_test_pw -e POSTGRES_DB=opt_test \
* -p 15433:5432 postgres:16-alpine
* SOURCE_REGISTRY_TEST_DATABASE_URL=postgres://opt:opt_test_pw@localhost:15433/opt_test \
* pnpm vitest run tests/p0/source-registry-postgres.realinfra.test.ts
*
* This is NOT a mock-repository test -- every assertion below is driven
* through SourceRegistryService with `databaseUrl` set, i.e. the real
* PostgresSourceRepository/PostgresSourceRegistryRepository HTTP-free SQL
* path (postgres.js against the live container), with a handful of
* assertions cross-checked with direct SQL against the same database to
* prove the rows are really there, not an artifact of the service's own
* in-process bookkeeping.
*
* beforeAll applies sql/002_source_registry.sql itself (every statement is
* `CREATE TABLE/INDEX IF NOT EXISTS`, so this is idempotent and safe to run
* against an already-migrated database too) -- this test does not require
* the operator to have pre-run migrations.
*/
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import fs from "node:fs";
import path from "node:path";
import postgres from "postgres";
import { SourceRegistryService } from "@/src/services/source-registry-service";
import { closeRepositorySqlForTests } from "@/src/repositories/db";
const TEST_DATABASE_URL = process.env.SOURCE_REGISTRY_TEST_DATABASE_URL;
const run = TEST_DATABASE_URL ? describe : describe.skip;
let adminSql: ReturnType<typeof postgres> | undefined;
let reachable = false;
let unreachableReason = "";
beforeAll(async () => {
if (!TEST_DATABASE_URL) {
console.warn(
"[realinfra] SOURCE_REGISTRY_TEST_DATABASE_URL not set this run -- skipping honestly, not fabricating success.",
);
return;
}
try {
adminSql = postgres(TEST_DATABASE_URL, { prepare: false, ssl: false, max: 2 });
await Promise.race([
adminSql`SELECT 1`,
new Promise((_, reject) => setTimeout(() => reject(new Error("connect timeout")), 5000)),
]);
const migrationPath = path.join(process.cwd(), "sql", "002_source_registry.sql");
const migration = fs.readFileSync(migrationPath, "utf8");
await adminSql.unsafe(migration);
reachable = true;
} catch (err) {
reachable = false;
unreachableReason = err instanceof Error ? err.message : String(err);
}
}, 30000);
afterAll(async () => {
await adminSql?.end({ timeout: 5 });
await closeRepositorySqlForTests();
});
function svc(): SourceRegistryService {
return new SourceRegistryService({ databaseUrl: TEST_DATABASE_URL });
}
run("SourceRegistryService against a LIVE Postgres (sql/002_source_registry.sql)", () => {
it("persists across independent service instances -- the exact bug this migration fixes (in-memory Maps were per-instance, not even per-process)", async (ctx) => {
if (!reachable) {
console.warn(
`[realinfra] Postgres not reachable this run (${unreachableReason}) -- skipping honestly.`,
);
ctx.skip();
return;
}
const stamp = Date.now();
const sourceId = `src_realpg_${stamp}`;
const tenantId = `t_realpg_${stamp}`;
const writer = svc();
await writer.registerSource({
source_id: sourceId,
tenant_id: tenantId,
project_id: "p_1",
source_type: "api",
name: "Live Postgres pilot source",
access: "api",
created_by: "realinfra-test",
rights: {
license_id: "official-public-data",
allowed_purposes: ["company_enrichment"],
pii_allowed: false,
redistribution: "customer_deliverable",
verified_at: "2026-09-01",
verified_by: "realinfra-test",
},
provenance: {
method: "official_api",
authority: "Bolagsverket",
cadence: "daily",
freshness_sla_hours: 24,
},
});
// A BRAND NEW SourceRegistryService, own SourceService/repository
// construction under the hood, same DATABASE_URL -- pre-migration this
// would see NOTHING (each `new SourceService()` owned a private Map).
const reader = svc();
const descriptor = await reader.describeSource(sourceId, tenantId);
expect(descriptor.source.source_id).toBe(sourceId);
expect(descriptor.source.tenant_id).toBe(tenantId);
expect(descriptor.rights?.license_id).toBe("official-public-data");
expect(descriptor.rights?.allowed_purposes).toContain("company_enrichment");
expect(descriptor.provenance?.method).toBe("official_api");
expect(descriptor.provenance?.authority).toBe("Bolagsverket");
// Cross-checked with direct SQL -- real rows, not in-process bookkeeping.
const sourceRows = await adminSql!<{ source_id: string; tenant_id: string }[]>`
SELECT source_id, tenant_id FROM opt_sources WHERE source_id = ${sourceId}`;
expect(sourceRows).toHaveLength(1);
expect(sourceRows[0].tenant_id).toBe(tenantId);
const rightsRows = await adminSql!<{ license_id: string; allowed_purposes: string[] }[]>`
SELECT license_id, allowed_purposes FROM opt_source_rights WHERE source_id = ${sourceId}`;
expect(rightsRows).toHaveLength(1);
expect(rightsRows[0].license_id).toBe("official-public-data");
expect(rightsRows[0].allowed_purposes).toEqual(["company_enrichment"]);
});
it("registering the same source_id twice -> 'already exists', enforced by real Postgres (not app-side memory)", async (ctx) => {
if (!reachable) return ctx.skip();
const stamp = Date.now();
const sourceId = `src_realpg_dup_${stamp}`;
const tenantId = `t_realpg_dup_${stamp}`;
await svc().registerSource({
source_id: sourceId,
tenant_id: tenantId,
project_id: "p_1",
source_type: "web",
name: "First",
});
await expect(
svc().registerSource({
source_id: sourceId,
tenant_id: tenantId,
project_id: "p_1",
source_type: "web",
name: "Second",
}),
).rejects.toThrow(/already exists/i);
});
it("cross-tenant describe -> NOT_FOUND against real Postgres (§50, no existence oracle)", async (ctx) => {
if (!reachable) return ctx.skip();
const stamp = Date.now();
const sourceId = `src_realpg_tenant_${stamp}`;
await svc().registerSource({
source_id: sourceId,
tenant_id: `t_a_realpg_${stamp}`,
project_id: "p_1",
source_type: "web",
name: "Tenant A source",
});
await expect(svc().describeSource(sourceId, `t_b_realpg_${stamp}`)).rejects.toMatchObject({
code: "NOT_FOUND",
});
// And a foreign-tenant query never sees it either.
const foreign = await svc().querySources({ tenant_id: `t_b_realpg_${stamp}` });
expect(foreign.find((s) => s.source_id === sourceId)).toBeUndefined();
});
it("purpose gate fail-closed against real Postgres: no rights row denies every purpose", async (ctx) => {
if (!reachable) return ctx.skip();
const stamp = Date.now();
const sourceId = `src_realpg_norights_${stamp}`;
const tenantId = `t_realpg_${stamp}`;
await svc().registerSource({
source_id: sourceId,
tenant_id: tenantId,
project_id: "p_1",
source_type: "web",
name: "No rights yet",
});
await expect(svc().assertPurposeAllowed(sourceId, tenantId, "anything")).rejects.toMatchObject({
code: "AUTHORIZATION_DENIED",
});
});
it("health checks: bounded to the latest 100 rows in Postgres, consecutive-failure count survives across service instances", async (ctx) => {
if (!reachable) return ctx.skip();
const stamp = Date.now();
const sourceId = `src_realpg_health_${stamp}`;
const tenantId = `t_realpg_${stamp}`;
await svc().registerSource({
source_id: sourceId,
tenant_id: tenantId,
project_id: "p_1",
source_type: "api",
name: "Health test source",
});
// 103 "down" checks (oldest -> newest), each via a FRESH service
// instance, simulating separate route-handler invocations.
for (let i = 0; i < 103; i++) {
await svc().recordHealthCheck({
source_id: sourceId,
tenant_id: tenantId,
status: "down",
checked_at: new Date(Date.now() - (300 - i) * 1000).toISOString(),
});
}
// Then 2 "up" checks, strictly more recent than every "down" above.
for (let i = 0; i < 2; i++) {
await svc().recordHealthCheck({
source_id: sourceId,
tenant_id: tenantId,
status: "up",
checked_at: new Date(Date.now() - (2 - i) * 500).toISOString(),
latency_ms: 42,
});
}
// Bounded storage: never more than 100 rows for this source_id in Postgres.
const countRows = await adminSql!<{ n: number }[]>`
SELECT count(*)::int AS n FROM opt_source_health_checks WHERE source_id = ${sourceId}`;
expect(countRows[0].n).toBeLessThanOrEqual(100);
// All 103 "down" writes plus 2 "up" writes were made; some of the
// oldest "down" rows must have been pruned.
expect(countRows[0].n).toBeLessThan(105);
const health = await svc().getHealth(sourceId, tenantId);
expect(health.status).toBe("up");
expect(health.consecutive_failures).toBe(0);
const freshness = await svc().getFreshness(sourceId, tenantId);
expect(freshness.fresh).toBe(true);
expect(freshness.last_success_at).toBeDefined();
});
it("source.query filters by purpose and redistribution against real rows in opt_source_rights", async (ctx) => {
if (!reachable) return ctx.skip();
const stamp = Date.now();
const tenantId = `t_realpg_query_${stamp}`;
const sourceA = `src_realpg_a_${stamp}`;
const sourceB = `src_realpg_b_${stamp}`;
await svc().registerSource({
source_id: sourceA,
tenant_id: tenantId,
project_id: "p_1",
source_type: "api",
name: "A",
rights: {
license_id: "l1",
allowed_purposes: ["company_enrichment"],
pii_allowed: false,
redistribution: "customer_deliverable",
verified_at: "2026-09-01",
verified_by: "t",
},
});
await svc().registerSource({
source_id: sourceB,
tenant_id: tenantId,
project_id: "p_1",
source_type: "web",
name: "B",
rights: {
license_id: "l2",
allowed_purposes: ["compliance"],
pii_allowed: false,
redistribution: "internal",
verified_at: "2026-09-01",
verified_by: "t",
},
});
const enrichment = await svc().querySources({ tenant_id: tenantId, purpose: "company_enrichment" });
expect(enrichment.map((s) => s.source_id)).toEqual([sourceA]);
const internalOnly = await svc().querySources({ tenant_id: tenantId, redistribution: "internal" });
expect(internalOnly.map((s) => s.source_id)).toEqual([sourceB]);
});
it("setRights/setProvenance upsert in place (matching the Map.set replace semantics they replace)", async (ctx) => {
if (!reachable) return ctx.skip();
const stamp = Date.now();
const sourceId = `src_realpg_upsert_${stamp}`;
const tenantId = `t_realpg_${stamp}`;
const registry = svc();
await registry.registerSource({
source_id: sourceId,
tenant_id: tenantId,
project_id: "p_1",
source_type: "api",
name: "Upsert test",
});
await registry.setRights({
source_id: sourceId,
tenant_id: tenantId,
license_id: "v1",
allowed_purposes: ["compliance"],
pii_allowed: false,
redistribution: "internal",
verified_at: "2026-09-01",
verified_by: "t",
});
await registry.setRights({
source_id: sourceId,
tenant_id: tenantId,
license_id: "v2",
allowed_purposes: ["compliance", "company_enrichment"],
pii_allowed: false,
redistribution: "customer_deliverable",
verified_at: "2026-09-02",
verified_by: "t2",
});
const rows = await adminSql!<
{ license_id: string }[]
>`SELECT license_id FROM opt_source_rights WHERE source_id = ${sourceId}`;
expect(rows).toHaveLength(1); // one row, updated in place -- not two
expect(rows[0].license_id).toBe("v2");
const rights = await registry.getRights(sourceId, tenantId);
expect(rights?.license_id).toBe("v2");
expect(rights?.allowed_purposes).toEqual(["compliance", "company_enrichment"]);
});
});
+6 -4
View File
@@ -75,9 +75,11 @@ describe("SourceRegistryService — purpose gate (fail closed)", () => {
it("denies a purpose outside the allowlist", async () => {
const registry = svc();
await registerBase(registry);
await expect(registry.assertPurposeAllowed("src_bolagsverket", "t_a", "marketing")).rejects.toMatchObject({
code: "AUTHORIZATION_DENIED",
});
await expect(registry.assertPurposeAllowed("src_bolagsverket", "t_a", "marketing")).rejects.toMatchObject(
{
code: "AUTHORIZATION_DENIED",
},
);
});
it("denies EVERY purpose when no rights record exists (fail closed)", async () => {
@@ -129,7 +131,7 @@ describe("SourceRegistryService — health + freshness", () => {
await registerBase(registry);
const old = new Date(Date.now() - 72 * 3_600_000).toISOString(); // 72h > 48h SLA
await registry.recordSuccess("src_bolagsverket", "t_a", old);
const freshness = registry.getFreshness("src_bolagsverket");
const freshness = await registry.getFreshness("src_bolagsverket", "t_a");
expect(freshness.fresh).toBe(false);
expect(freshness.age_hours).toBeGreaterThan(48);
});