diff --git a/app/api/0pt/sources/[sourceId]/freshness/route.ts b/app/api/0pt/sources/[sourceId]/freshness/route.ts new file mode 100644 index 0000000..24ae310 --- /dev/null +++ b/app/api/0pt/sources/[sourceId]/freshness/route.ts @@ -0,0 +1,27 @@ +/** + * app/api/0pt/sources/[sourceId]/freshness/route.ts + * ===================================================== + * Freshness view derived from provenance's freshness_sla_hours + the last + * successful observation (recorded via POST .../health with status "up", + * or SourceRegistryService.recordSuccess directly). Read-only — there is no + * separate write path here by design; freshness is a derived signal, not + * its own record. + */ +import { NextRequest, NextResponse } from "next/server"; +import { sourceRegistryService } from "@/src/services/source-registry"; +import { siaxErrorResponse } from "@/src/http/siax-error-response"; + +export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) { + try { + const { sourceId } = await params; + const { searchParams } = new URL(request.url); + const tenantId = searchParams.get("tenant_id"); + if (!tenantId) { + return NextResponse.json({ error: "tenant_id is required" }, { status: 400 }); + } + const freshness = await sourceRegistryService.getFreshness(sourceId, tenantId); + return NextResponse.json(freshness); + } catch (error: unknown) { + return siaxErrorResponse(error); + } +} diff --git a/app/api/0pt/sources/[sourceId]/health/route.ts b/app/api/0pt/sources/[sourceId]/health/route.ts new file mode 100644 index 0000000..c376063 --- /dev/null +++ b/app/api/0pt/sources/[sourceId]/health/route.ts @@ -0,0 +1,50 @@ +/** + * app/api/0pt/sources/[sourceId]/health/route.ts + * ================================================== + * source.health — record a health observation (POST) and read the + * aggregate view (GET): status, last check, consecutive failures. Bounded + * history (latest 100 observations) — this is an operational signal, not + * an audit log; AUD0 owns history. + */ +import { NextRequest, NextResponse } from "next/server"; +import { parseBody, recordSourceHealthCheckSchema } from "@/src/validation/0pt-schemas"; +import { sourceRegistryService } from "@/src/services/source-registry"; +import { siaxErrorResponse } from "@/src/http/siax-error-response"; + +export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) { + try { + const { sourceId } = await params; + const { searchParams } = new URL(request.url); + const tenantId = searchParams.get("tenant_id"); + if (!tenantId) { + return NextResponse.json({ error: "tenant_id is required" }, { status: 400 }); + } + const health = await sourceRegistryService.getHealth(sourceId, tenantId); + return NextResponse.json(health); + } catch (error: unknown) { + return siaxErrorResponse(error); + } +} + +export async function POST(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) { + try { + const { sourceId } = await params; + const __raw = await request.json(); + const __parsed = parseBody(recordSourceHealthCheckSchema, __raw); + if (!__parsed.ok) { + return NextResponse.json(__parsed.body, { status: __parsed.status }); + } + const body = __parsed.data; + const health = await sourceRegistryService.recordHealthCheck({ + source_id: sourceId, + tenant_id: body.tenant_id, + status: body.status, + checked_at: body.checked_at ?? new Date().toISOString(), + latency_ms: body.latency_ms, + detail: body.detail, + }); + return NextResponse.json(health, { status: 201 }); + } catch (error: unknown) { + return siaxErrorResponse(error); + } +} diff --git a/app/api/0pt/sources/[sourceId]/provenance/route.ts b/app/api/0pt/sources/[sourceId]/provenance/route.ts new file mode 100644 index 0000000..0628179 --- /dev/null +++ b/app/api/0pt/sources/[sourceId]/provenance/route.ts @@ -0,0 +1,52 @@ +/** + * app/api/0pt/sources/[sourceId]/provenance/route.ts + * ====================================================== + * Per-SOURCE provenance/method metadata (Master-DoD §10) — HOW a source is + * obtained (official_api | bulk_download | authorized_web | licensed_feed | + * manual_upload), not per-field bridge_provenance. See BEFORE_STATE.md §3 + * for that distinction: this route does not, and cannot, close the + * bridge_provenance gap — it only surfaces what SourceRegistryService + * already models at the source level. + */ +import { NextRequest, NextResponse } from "next/server"; +import { parseBody, setSourceProvenanceSchema } from "@/src/validation/0pt-schemas"; +import { sourceRegistryService } from "@/src/services/source-registry"; +import { siaxErrorResponse } from "@/src/http/siax-error-response"; + +export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) { + try { + const { sourceId } = await params; + const { searchParams } = new URL(request.url); + const tenantId = searchParams.get("tenant_id"); + if (!tenantId) { + return NextResponse.json({ error: "tenant_id is required" }, { status: 400 }); + } + const provenance = await sourceRegistryService.getProvenance(sourceId, tenantId); + if (!provenance) { + return NextResponse.json({ error: `source ${sourceId} has no provenance record` }, { status: 404 }); + } + return NextResponse.json(provenance); + } catch (error: unknown) { + return siaxErrorResponse(error); + } +} + +export async function PUT(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) { + try { + const { sourceId } = await params; + const __raw = await request.json(); + const __parsed = parseBody(setSourceProvenanceSchema, __raw); + if (!__parsed.ok) { + return NextResponse.json(__parsed.body, { status: __parsed.status }); + } + const body = __parsed.data; + const provenance = await sourceRegistryService.setProvenance({ + ...body, + source_id: sourceId, + updated_at: new Date().toISOString(), + }); + return NextResponse.json(provenance); + } catch (error: unknown) { + return siaxErrorResponse(error); + } +} diff --git a/app/api/0pt/sources/[sourceId]/query/route.ts b/app/api/0pt/sources/[sourceId]/query/route.ts index 9d97140..b9afd94 100644 --- a/app/api/0pt/sources/[sourceId]/query/route.ts +++ b/app/api/0pt/sources/[sourceId]/query/route.ts @@ -4,27 +4,24 @@ * 0pt.query_source — bounded synchronous query execution for a registered * Source. Additive: does not modify sibling ../route.ts or ../../route.ts. * - * NOTE — in-memory scope: this module instantiates its own SourceService, - * matching the existing sources/route.ts and sources/[sourceId]/route.ts - * pattern (each API route module owns a fresh in-memory registry, so state - * is not shared across route modules yet — see the "Singleton note" in - * src/services/document-capture-service.ts for prior art on this same repo - * limitation). A source registered via POST /api/0pt/sources will not be - * resolvable here until SourceService is backed by shared persistence. - * Correction: this comment previously claimed `sql/002_p0_dataset_quality.sql` - * already defined an `opt_sources` table for that. That file does not exist - * and never has (git log --all: zero commits, any branch, ever). sql/ today - * has only 001_init.sql and 004_iceberg_write_boundary.sql. The real fix — - * backing SourceService with real persistence — is still open; no migration - * for it exists yet. - * Details: docs/P0_QUERY_SOURCE_FLOW.md. + * UPDATED (Source Registry Postgres-persistence pass): now imports the + * shared `sourceService` singleton from src/services/source-registry.ts + * instead of constructing its own SourceService, closing the "not even a + * shared process-wide singleton" gap BEFORE_STATE.md §1 flagged (a source + * registered via POST /api/0pt/sources used to be invisible here). With + * DATABASE_URL configured this is doubly fixed — state lives in Postgres + * (sql/002_source_registry.sql), genuinely shared across processes, not + * just within one. Details: docs/P0_QUERY_SOURCE_FLOW.md. */ import { NextRequest, NextResponse } from "next/server"; import { parseBody, sourceQuerySchema } from "@/src/validation/0pt-schemas"; -import { SourceService } from "@/src/services/source-service"; -import { SourceQueryService, SourceQueryTimeoutError, SourceQueryUnsupportedError } from "@/src/services/source-query-service"; +import { sourceService } from "@/src/services/source-registry"; +import { + SourceQueryService, + SourceQueryTimeoutError, + SourceQueryUnsupportedError, +} from "@/src/services/source-query-service"; -const sourceService = new SourceService(); const queryService = new SourceQueryService({ sourceService }); export async function POST(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) { diff --git a/app/api/0pt/sources/[sourceId]/rights/route.ts b/app/api/0pt/sources/[sourceId]/rights/route.ts new file mode 100644 index 0000000..05c3708 --- /dev/null +++ b/app/api/0pt/sources/[sourceId]/rights/route.ts @@ -0,0 +1,45 @@ +/** + * app/api/0pt/sources/[sourceId]/rights/route.ts + * ================================================== + * Rights / license / purpose metadata (Master-DoD §10). PUT is an upsert + * (SourceRegistryService.setRights), matching the fail-closed purpose gate: + * no rights record at all denies every purpose until one is set here. + */ +import { NextRequest, NextResponse } from "next/server"; +import { parseBody, setSourceRightsSchema } from "@/src/validation/0pt-schemas"; +import { sourceRegistryService } from "@/src/services/source-registry"; +import { siaxErrorResponse } from "@/src/http/siax-error-response"; + +export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) { + try { + const { sourceId } = await params; + const { searchParams } = new URL(request.url); + const tenantId = searchParams.get("tenant_id"); + if (!tenantId) { + return NextResponse.json({ error: "tenant_id is required" }, { status: 400 }); + } + const rights = await sourceRegistryService.getRights(sourceId, tenantId); + if (!rights) { + return NextResponse.json({ error: `source ${sourceId} has no rights record` }, { status: 404 }); + } + return NextResponse.json(rights); + } catch (error: unknown) { + return siaxErrorResponse(error); + } +} + +export async function PUT(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) { + try { + const { sourceId } = await params; + const __raw = await request.json(); + const __parsed = parseBody(setSourceRightsSchema, __raw); + if (!__parsed.ok) { + return NextResponse.json(__parsed.body, { status: __parsed.status }); + } + const body = __parsed.data; + const rights = await sourceRegistryService.setRights({ ...body, source_id: sourceId }); + return NextResponse.json(rights); + } catch (error: unknown) { + return siaxErrorResponse(error); + } +} diff --git a/app/api/0pt/sources/[sourceId]/route.ts b/app/api/0pt/sources/[sourceId]/route.ts new file mode 100644 index 0000000..9397175 --- /dev/null +++ b/app/api/0pt/sources/[sourceId]/route.ts @@ -0,0 +1,26 @@ +/** + * app/api/0pt/sources/[sourceId]/route.ts + * ========================================== + * source.describe — full SourceDescriptor (base + rights + provenance + + * health + freshness + rate policy). Cross-tenant probes get 404 NOT_FOUND + * (no existence oracle, §50) via SourceRegistryService.describeSource's + * assertSourceTenant gate. + */ +import { NextRequest, NextResponse } from "next/server"; +import { sourceRegistryService } from "@/src/services/source-registry"; +import { siaxErrorResponse } from "@/src/http/siax-error-response"; + +export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) { + try { + const { sourceId } = await params; + const { searchParams } = new URL(request.url); + const tenantId = searchParams.get("tenant_id"); + if (!tenantId) { + return NextResponse.json({ error: "tenant_id is required" }, { status: 400 }); + } + const descriptor = await sourceRegistryService.describeSource(sourceId, tenantId); + return NextResponse.json(descriptor); + } catch (error: unknown) { + return siaxErrorResponse(error); + } +} diff --git a/app/api/0pt/sources/rate-policies/[sourceRatePolicyId]/route.ts b/app/api/0pt/sources/rate-policies/[sourceRatePolicyId]/route.ts index 245038a..47db8f3 100644 --- a/app/api/0pt/sources/rate-policies/[sourceRatePolicyId]/route.ts +++ b/app/api/0pt/sources/rate-policies/[sourceRatePolicyId]/route.ts @@ -1,9 +1,10 @@ import { NextResponse } from "next/server"; -import { SourceRatePolicyService } from "@/src/services/source-rate-policy-service"; +import { sourceRatePolicyService as service } from "@/src/services/source-registry"; -const service = new SourceRatePolicyService(); - -export async function GET(_request: Request, { params }: { params: Promise<{ sourceRatePolicyId: string }> }) { +export async function GET( + _request: Request, + { params }: { params: Promise<{ sourceRatePolicyId: string }> }, +) { try { const { sourceRatePolicyId } = await params; const policy = await service.getSourceRatePolicy(sourceRatePolicyId); diff --git a/app/api/0pt/sources/rate-policies/route.ts b/app/api/0pt/sources/rate-policies/route.ts index caa8607..3c78ffc 100644 --- a/app/api/0pt/sources/rate-policies/route.ts +++ b/app/api/0pt/sources/rate-policies/route.ts @@ -1,8 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; import { parseBody, createSourceRatePolicySchema } from "@/src/validation/0pt-schemas"; -import { SourceRatePolicyService } from "@/src/services/source-rate-policy-service"; - -const service = new SourceRatePolicyService(); +import { sourceRatePolicyService as service } from "@/src/services/source-registry"; export async function POST(request: NextRequest) { try { diff --git a/app/api/0pt/sources/route.ts b/app/api/0pt/sources/route.ts new file mode 100644 index 0000000..62c2051 --- /dev/null +++ b/app/api/0pt/sources/route.ts @@ -0,0 +1,55 @@ +/** + * app/api/0pt/sources/route.ts + * ============================== + * source.register (POST) + source.query (GET) — the P0 Source Registry + * route surface BEFORE_STATE.md §1 found entirely missing + * (`grep -rl "SourceRegistryService" app/` returned nothing). Backed by + * SourceRegistryService, which now persists to Postgres when DATABASE_URL + * is configured (src/repositories/*, sql/002_source_registry.sql). + * + * Uses the shared singleton from src/services/source-registry.ts, not a + * fresh `new SourceRegistryService()` per route module — see that file's + * doc comment. + */ +import { NextRequest, NextResponse } from "next/server"; +import { parseBody, registerSourceSchema } from "@/src/validation/0pt-schemas"; +import { sourceRegistryService } from "@/src/services/source-registry"; +import { siaxErrorResponse } from "@/src/http/siax-error-response"; +import type { SourceStatus, SourceType } from "@/src/types/source"; +import type { SourceRights } from "@/src/types/source-registry"; + +export async function POST(request: NextRequest) { + try { + const __raw = await request.json(); + const __parsed = parseBody(registerSourceSchema, __raw); + if (!__parsed.ok) { + return NextResponse.json(__parsed.body, { status: __parsed.status }); + } + const body = __parsed.data; + const descriptor = await sourceRegistryService.registerSource(body); + return NextResponse.json(descriptor, { status: 201 }); + } catch (error: unknown) { + return siaxErrorResponse(error); + } +} + +export async function GET(request: NextRequest) { + try { + const { searchParams } = new URL(request.url); + const tenantId = searchParams.get("tenant_id"); + if (!tenantId) { + return NextResponse.json({ error: "tenant_id is required" }, { status: 400 }); + } + const sources = await sourceRegistryService.querySources({ + tenant_id: tenantId, + project_id: searchParams.get("project_id") ?? undefined, + source_type: (searchParams.get("source_type") as SourceType) ?? undefined, + status: (searchParams.get("status") as SourceStatus) ?? undefined, + purpose: searchParams.get("purpose") ?? undefined, + redistribution: (searchParams.get("redistribution") as SourceRights["redistribution"]) ?? undefined, + }); + return NextResponse.json(sources); + } catch (error: unknown) { + return siaxErrorResponse(error); + } +} diff --git a/sql/002_source_registry.sql b/sql/002_source_registry.sql new file mode 100644 index 0000000..d3a990d --- /dev/null +++ b/sql/002_source_registry.sql @@ -0,0 +1,105 @@ +-- 0pt Source Registry persistence (Master-DoD §10; BEFORE_STATE.md §1). +-- +-- Backs src/services/source-service.ts (0pt.source.schema.v1, the base +-- registry) and src/services/source-registry-service.ts (rights/license/ +-- purpose, provenance, health, freshness — the P0 Source Registry surface +-- layered on top). Both previously held their state in bare `new Map()`s, +-- scoped per service *instance* (not even shared within one process) and +-- hard-blocked from production boot by resolveStoreMode +-- (src/platform/dependency-mode.ts) for lack of a DB. This migration is the +-- DB half of closing that gap; src/repositories/* is the code half. +-- +-- Deliberately numbered 002, not 006: sql/004_iceberg_write_boundary.sql's +-- opt_source_rate_policies table already carries +-- `source_id TEXT NOT NULL REFERENCES opt_sources (source_id)` — a +-- forward reference to a table that has never existed in any sql/ file +-- (opt_sources was, until this file, only ever a doc-comment aspiration — +-- see the "What Postgres migration would require" section of +-- BEFORE_STATE.md §1 and the retracted `sql/002_p0_dataset_quality.sql` +-- claim in CLAUDE.md/app/api/0pt/sources/[sourceId]/query/route.ts, an +-- UNRELATED fabricated claim about a different, never-written file that +-- happened to reuse this same number). Applying 004 before opt_sources +-- exists would fail outright on a fresh database; this file must run +-- between 001 and 004 to make 004's existing FK resolvable. This is a +-- pre-existing ordering bug in 004, not introduced here — fixed here only +-- because this pass needed opt_sources to exist anyway. + +-- ── 0pt.source.schema.v1 (src/types/source.ts) ───────────────────────── +CREATE TABLE IF NOT EXISTS opt_sources ( + source_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + source_type TEXT NOT NULL, -- database | file | web | api | stream | manual + name TEXT NOT NULL, + adapter_id TEXT, -- e.g. "fabric_onelake_sql" (src/adapters/fabric-onelake-adapter.ts) + access TEXT, -- api | bulk | authorized_web | licensed (never a bypass mode) + description TEXT, + status TEXT NOT NULL DEFAULT 'active', -- active | paused | archived | deprecated + correlation_id TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ, + created_by TEXT +); +CREATE INDEX IF NOT EXISTS opt_sources_tenant_project_idx ON opt_sources (tenant_id, project_id); +CREATE INDEX IF NOT EXISTS opt_sources_type_idx ON opt_sources (source_type); +CREATE INDEX IF NOT EXISTS opt_sources_status_idx ON opt_sources (status); + +-- ── Rights / license / purpose metadata (src/types/source-registry.ts) ─ +-- One row per source (SourceRegistryService.setRights is an upsert-by- +-- source_id, same as the Map it replaces). No rights row = fail-closed +-- deny in assertPurposeAllowed — that logic lives in the service, not here. +CREATE TABLE IF NOT EXISTS opt_source_rights ( + source_id TEXT PRIMARY KEY REFERENCES opt_sources (source_id), + tenant_id TEXT NOT NULL, + license_id TEXT NOT NULL, + license_url TEXT, + allowed_purposes TEXT[] NOT NULL DEFAULT '{}', + pii_allowed BOOLEAN NOT NULL DEFAULT false, + redistribution TEXT NOT NULL, -- internal | customer_deliverable | public | prohibited + attribution TEXT, + verified_at TEXT NOT NULL, -- ISO date the assessment was made (date-only in practice; kept as TEXT, not TIMESTAMPTZ, so it round-trips byte-for-byte) + verified_by TEXT NOT NULL, + notes TEXT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS opt_source_rights_tenant_idx ON opt_source_rights (tenant_id); + +-- ── Provenance / method metadata (per-SOURCE, not per-field — see +-- BEFORE_STATE.md §3 for the distinction from bridge_provenance) ──────── +CREATE TABLE IF NOT EXISTS opt_source_provenance ( + source_id TEXT PRIMARY KEY REFERENCES opt_sources (source_id), + tenant_id TEXT NOT NULL, + method TEXT NOT NULL, -- official_api | bulk_download | authorized_web | licensed_feed | manual_upload + authority TEXT, + compliance_evidence_ref TEXT, + cadence TEXT, + freshness_sla_hours NUMERIC, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS opt_source_provenance_tenant_idx ON opt_source_provenance (tenant_id); + +-- ── Health observations (source.health) — an operational signal, NOT an +-- audit log (AUD0 owns history). Bounded to the latest 100 rows per +-- source_id by the repository layer (src/repositories/postgres-source- +-- registry-repository.ts), mirroring the in-memory implementation's +-- `list.splice(0, list.length - 100)` bound. +CREATE TABLE IF NOT EXISTS opt_source_health_checks ( + health_check_id BIGSERIAL PRIMARY KEY, + source_id TEXT NOT NULL REFERENCES opt_sources (source_id), + tenant_id TEXT NOT NULL, + status TEXT NOT NULL, -- up | degraded | down | unknown + checked_at TIMESTAMPTZ NOT NULL, + latency_ms INT, + detail TEXT -- short operator-facing text; never a raw upstream body +); +CREATE INDEX IF NOT EXISTS opt_source_health_checks_source_idx ON opt_source_health_checks (source_id, checked_at DESC, health_check_id DESC); + +-- ── Freshness: last successful observation per source (source.health's +-- "up" checks call recordSuccess; getFreshness reads this + provenance's +-- freshness_sla_hours). One row per source — upserted, not appended. +CREATE TABLE IF NOT EXISTS opt_source_freshness ( + source_id TEXT PRIMARY KEY REFERENCES opt_sources (source_id), + tenant_id TEXT NOT NULL, + last_success_at TIMESTAMPTZ NOT NULL +); +CREATE INDEX IF NOT EXISTS opt_source_freshness_tenant_idx ON opt_source_freshness (tenant_id); diff --git a/sql/004_iceberg_write_boundary.sql b/sql/004_iceberg_write_boundary.sql index 3a27665..bb03580 100644 --- a/sql/004_iceberg_write_boundary.sql +++ b/sql/004_iceberg_write_boundary.sql @@ -8,6 +8,18 @@ -- different service's database. 0pt holds only the reference, never a copy -- of ST0RE's table/catalog/snapshot state (see the module doc comment on -- src/clients/st0re-lakehouse-client.ts). +-- +-- Ordering note (added by the Source Registry Postgres-persistence pass): +-- opt_source_rate_policies below has always forward-referenced +-- `opt_sources (source_id)`, a table no sql/ file defined until +-- sql/002_source_registry.sql landed -- applying this file to a fresh +-- database before 002 would fail on that FK. 002 is now the one that must +-- run first (001 -> 002 -> 004 -> 005). Flagging, not fixing here: this +-- file's OTHER forward reference, `opt_dataset_iceberg_mappings.dataset_id +-- REFERENCES opt_datasets (dataset_id)` below, has the identical problem -- +-- no sql/ file defines opt_datasets either -- and remains open; it is a +-- dataset-registry gap, unrelated to and out of scope for the Source +-- Registry work that added 002. CREATE TABLE IF NOT EXISTS opt_dataset_iceberg_mappings ( dataset_iceberg_mapping_id TEXT PRIMARY KEY, diff --git a/src/http/siax-error-response.ts b/src/http/siax-error-response.ts new file mode 100644 index 0000000..2547a75 --- /dev/null +++ b/src/http/siax-error-response.ts @@ -0,0 +1,25 @@ +/** + * src/http/siax-error-response.ts + * ================================= + * Maps a thrown error to a NextResponse for the source-registry routes + * (app/api/0pt/sources/*). SourceRegistryService throws real `SiaxError` + * instances (src/platform/errors.ts) with a pinned `.httpStatus`/`.code` — + * this honors that mapping directly instead of re-deriving status codes from + * regex-matching `error.message`, the ad-hoc pattern most other 0pt routes + * use (app/api/0pt/documents/[documentId]/extraction/route.ts etc.) because + * their services still throw plain `Error`. SourceService/SourceRegistryService's + * lower-level plain Errors ("already exists", "not found") still fall back + * to that same regex convention here, for consistency with the rest of the + * repo's error surface. + */ +import { NextResponse } from "next/server"; +import { SiaxError } from "../platform/errors"; + +export function siaxErrorResponse(error: unknown): NextResponse { + if (error instanceof SiaxError) { + return NextResponse.json(error.toJSON(), { status: error.httpStatus }); + } + const message = error instanceof Error ? error.message : "Unknown error"; + const status = /already exists/i.test(message) ? 409 : /not found/i.test(message) ? 404 : 400; + return NextResponse.json({ error: message }, { status }); +} diff --git a/src/repositories/db.ts b/src/repositories/db.ts new file mode 100644 index 0000000..e32acf6 --- /dev/null +++ b/src/repositories/db.ts @@ -0,0 +1,61 @@ +/** + * src/repositories/db.ts + * ======================== + * Lazy Postgres connection for P0 capability repositories (src/repositories/*). + * + * Deliberately separate from lib/db.ts: that module is tuned for the + * Vercel-hosted edge Postgres behind app/api/orders + app/api/subscriptions + * (`ssl: "require"`, because that's what Vercel Postgres needs) and lives in + * lib/, which src/ (the blocking-typechecked capability layer, + * .dependency-cruiser.cjs's "no-src-importing-app-or-components" rule's + * sibling concern) does not depend on. The P0 capability layer's Postgres + * instance is the self-hosted VPS Postgres from docs/08-infrastructure.md + * (infra/docker-compose.yml's `postgres` service, same private docker network + * as the FastAPI engine, no TLS terminated in front of it) -- forcing + * `ssl: "require"` here would break every real connection to it. SSL is + * opt-in via PGSSL=require for deployments that do put TLS in front. + * + * Same lazy/env-guarded shape as lib/db.ts, and the same production-safety + * decision as every other P0 dependency: src/platform/dependency-mode.ts's + * `resolveStoreMode` is what actually decides postgres-vs-memory (and fails + * hard in production with no URL) -- this module only opens the connection + * once that decision has already been made. + */ +import postgres from "postgres"; + +export type RepositorySql = ReturnType; + +/** DATABASE_URL, falling back to Vercel Postgres's env names -- same precedence lib/db.ts's dbUrl() uses, kept independent so src/ has no import on lib/. */ +export function repositoryDatabaseUrl(env: NodeJS.ProcessEnv = process.env): string | undefined { + return env.DATABASE_URL || env.POSTGRES_URL || env.POSTGRES_PRISMA_URL; +} + +let _sql: RepositorySql | null = null; +let _urlUsed: string | undefined; + +/** Returns a memoized connection for `url`; re-opens if a different URL is requested (e.g. tests pointing at a fresh testcontainer). */ +export function getRepositorySql(url: string): RepositorySql { + if (_sql && _urlUsed === url) return _sql; + _sql = postgres(url, { + prepare: false, + idle_timeout: 20, + max: 5, + ssl: process.env.PGSSL === "require" ? "require" : false, + }); + _urlUsed = url; + return _sql; +} + +/** Test-only escape hatch: force the next getRepositorySql call to open a fresh connection. */ +export function resetRepositorySqlForTests(): void { + _sql = null; + _urlUsed = undefined; +} + +/** Test-only: gracefully close the memoized connection (so a vitest run doesn't hang on an open socket) and clear it. */ +export async function closeRepositorySqlForTests(): Promise { + const sql = _sql; + _sql = null; + _urlUsed = undefined; + if (sql) await sql.end({ timeout: 5 }); +} diff --git a/src/repositories/in-memory-source-registry-repository.ts b/src/repositories/in-memory-source-registry-repository.ts new file mode 100644 index 0000000..a0cf046 --- /dev/null +++ b/src/repositories/in-memory-source-registry-repository.ts @@ -0,0 +1,59 @@ +/** + * src/repositories/in-memory-source-registry-repository.ts + * ============================================================ + * The original SourceRegistryService storage (four Maps: rights, + * provenance, healthChecks, lastSuccess), extracted verbatim behind the + * SourceRegistryRepository seam. Health-check bounding (latest 100 + * observations) is preserved exactly; `listHealthChecks` returns + * newest-first to match the Postgres implementation's `ORDER BY checked_at + * DESC` contract (the original Map-based `getHealth` read `list[list.length + * - 1]` for "most recent" and scanned backward for consecutive failures -- + * callers now get the same data pre-ordered instead). + */ +import type { SourceHealthCheck, SourceProvenance, SourceRights } from "../types/source-registry"; +import type { SourceRegistryRepository } from "./source-registry-repository"; + +export class InMemorySourceRegistryRepository implements SourceRegistryRepository { + private readonly rights = new Map(); + private readonly provenance = new Map(); + private readonly healthChecks = new Map(); + private readonly lastSuccess = new Map(); + + async setRights(rights: SourceRights): Promise { + this.rights.set(rights.source_id, rights); + } + + async getRights(sourceId: string): Promise { + return this.rights.get(sourceId); + } + + async setProvenance(prov: SourceProvenance): Promise { + this.provenance.set(prov.source_id, prov); + } + + async getProvenance(sourceId: string): Promise { + return this.provenance.get(sourceId); + } + + async recordSuccess(sourceId: string, _tenantId: string, atIso: string): Promise { + this.lastSuccess.set(sourceId, atIso); + } + + async getLastSuccess(sourceId: string): Promise { + return this.lastSuccess.get(sourceId); + } + + async appendHealthCheck(check: SourceHealthCheck): Promise { + const list = this.healthChecks.get(check.source_id) ?? []; + list.push(check); + // Bounded history -- health is an operational signal, not an audit log + // (AUD0 owns history); keep the latest 100 observations. + if (list.length > 100) list.splice(0, list.length - 100); + this.healthChecks.set(check.source_id, list); + } + + async listHealthChecks(sourceId: string, limit = 100): Promise { + const list = this.healthChecks.get(sourceId) ?? []; + return [...list].reverse().slice(0, limit); + } +} diff --git a/src/repositories/in-memory-source-repository.ts b/src/repositories/in-memory-source-repository.ts new file mode 100644 index 0000000..53babf9 --- /dev/null +++ b/src/repositories/in-memory-source-repository.ts @@ -0,0 +1,95 @@ +/** + * src/repositories/in-memory-source-repository.ts + * ================================================== + * The original SourceService storage, extracted verbatim behind the + * SourceRepository seam (src/repositories/source-repository.ts) so + * PostgresSourceRepository can run the exact same business rules + * ("already exists", "name is required", "not found", filter semantics) + * against real Postgres instead. Used outside production when no + * DATABASE_URL is configured (resolveStoreMode), and directly by tests that + * want a hermetic, dependency-free SourceService. + */ +import type { CreateSourceInput, Source, SourceStatus, SourceType } from "../types/source"; +import type { SourceListFilter, SourceRepository } from "./source-repository"; + +/** Storage shape: nullable columns, exactly mirroring `opt_sources`. */ +interface SourceRecord { + source_id: string; + tenant_id: string; + project_id: string; + source_type: string; + name: string; + adapter_id: string | null; + access: string | null; + description: string | null; + status: string; + correlation_id: string | null; + created_at: string; + updated_at: string | null; + created_by: string | null; +} + +export class InMemorySourceRepository implements SourceRepository { + private sources: Map = new Map(); + + async create(input: CreateSourceInput): Promise { + if (this.sources.has(input.source_id)) { + throw new Error(`Source ${input.source_id} already exists`); + } + if (!input.name) { + throw new Error("name is required"); + } + + const now = new Date().toISOString(); + const record: SourceRecord = { + source_id: input.source_id, + tenant_id: input.tenant_id, + project_id: input.project_id, + source_type: input.source_type, + name: input.name, + adapter_id: input.adapter_id ?? null, + access: input.access ?? null, + description: input.description ?? null, + status: input.status ?? "active", + correlation_id: input.correlation_id ?? null, + created_at: now, + updated_at: null, + created_by: input.created_by ?? null, + }; + this.sources.set(record.source_id, record); + return toSource(record); + } + + async get(sourceId: string): Promise { + const record = this.sources.get(sourceId); + if (!record) throw new Error(`Source ${sourceId} not found`); + return toSource(record); + } + + async list(filter?: SourceListFilter): Promise { + let results = Array.from(this.sources.values()); + if (filter?.tenant_id) results = results.filter((r) => r.tenant_id === filter.tenant_id); + if (filter?.project_id) results = results.filter((r) => r.project_id === filter.project_id); + if (filter?.source_type) results = results.filter((r) => r.source_type === filter.source_type); + if (filter?.status) results = results.filter((r) => r.status === filter.status); + return results.map(toSource); + } +} + +function toSource(r: SourceRecord): Source { + return { + source_id: r.source_id, + tenant_id: r.tenant_id, + project_id: r.project_id, + source_type: r.source_type as SourceType, + name: r.name, + adapter_id: r.adapter_id ?? undefined, + access: (r.access as Source["access"]) ?? undefined, + description: r.description ?? undefined, + status: r.status as SourceStatus, + correlation_id: r.correlation_id ?? undefined, + created_at: r.created_at, + updated_at: r.updated_at ?? undefined, + created_by: r.created_by ?? undefined, + }; +} diff --git a/src/repositories/postgres-source-registry-repository.ts b/src/repositories/postgres-source-registry-repository.ts new file mode 100644 index 0000000..e30abe3 --- /dev/null +++ b/src/repositories/postgres-source-registry-repository.ts @@ -0,0 +1,197 @@ +/** + * src/repositories/postgres-source-registry-repository.ts + * ========================================================== + * Real Postgres backing for SourceRegistryRepository, against + * `opt_source_rights` / `opt_source_provenance` / `opt_source_health_checks` + * / `opt_source_freshness` (sql/002_source_registry.sql). Rights and + * provenance are one-row-per-source, upserted (matching the Map.set + * replace-in-place semantics they replace); health checks are append-only, + * bounded to the latest 100 rows per source_id after every insert; freshness + * is a single upserted "last success" row per source. + */ +import type { + SourceHealthCheck, + SourceHealthStatus, + SourceProvenance, + SourceRights, +} from "../types/source-registry"; +import type { RepositorySql } from "./db"; +import type { SourceRegistryRepository } from "./source-registry-repository"; + +function toIsoString(v: string | Date): string { + return v instanceof Date ? v.toISOString() : v; +} + +interface RightsRow { + source_id: string; + tenant_id: string; + license_id: string; + license_url: string | null; + allowed_purposes: string[]; + pii_allowed: boolean; + redistribution: string; + attribution: string | null; + verified_at: string; + verified_by: string; + notes: string | null; +} + +function toRights(r: RightsRow): SourceRights { + return { + source_id: r.source_id, + tenant_id: r.tenant_id, + license_id: r.license_id, + license_url: r.license_url ?? undefined, + allowed_purposes: r.allowed_purposes ?? [], + pii_allowed: r.pii_allowed, + redistribution: r.redistribution as SourceRights["redistribution"], + attribution: r.attribution ?? undefined, + verified_at: r.verified_at, + verified_by: r.verified_by, + notes: r.notes ?? undefined, + }; +} + +interface ProvenanceRow { + source_id: string; + tenant_id: string; + method: string; + authority: string | null; + compliance_evidence_ref: string | null; + cadence: string | null; + freshness_sla_hours: string | number | null; + updated_at: string | Date; +} + +function toProvenance(r: ProvenanceRow): SourceProvenance { + return { + source_id: r.source_id, + tenant_id: r.tenant_id, + method: r.method as SourceProvenance["method"], + authority: r.authority ?? undefined, + compliance_evidence_ref: r.compliance_evidence_ref ?? undefined, + cadence: r.cadence ?? undefined, + freshness_sla_hours: r.freshness_sla_hours === null ? undefined : Number(r.freshness_sla_hours), + updated_at: toIsoString(r.updated_at), + }; +} + +interface HealthCheckRow { + source_id: string; + tenant_id: string; + status: string; + checked_at: string | Date; + latency_ms: number | null; + detail: string | null; +} + +function toHealthCheck(r: HealthCheckRow): SourceHealthCheck { + return { + source_id: r.source_id, + tenant_id: r.tenant_id, + status: r.status as SourceHealthStatus, + checked_at: toIsoString(r.checked_at), + latency_ms: r.latency_ms ?? undefined, + detail: r.detail ?? undefined, + }; +} + +export class PostgresSourceRegistryRepository implements SourceRegistryRepository { + constructor(private readonly sql: RepositorySql) {} + + async setRights(rights: SourceRights): Promise { + await this.sql` + INSERT INTO opt_source_rights + (source_id, tenant_id, license_id, license_url, allowed_purposes, pii_allowed, redistribution, attribution, verified_at, verified_by, notes, updated_at) + VALUES + (${rights.source_id}, ${rights.tenant_id}, ${rights.license_id}, ${rights.license_url ?? null}, + ${this.sql.array(rights.allowed_purposes)}, ${rights.pii_allowed}, ${rights.redistribution}, + ${rights.attribution ?? null}, ${rights.verified_at}, ${rights.verified_by}, ${rights.notes ?? null}, now()) + ON CONFLICT (source_id) DO UPDATE SET + tenant_id = EXCLUDED.tenant_id, + license_id = EXCLUDED.license_id, + license_url = EXCLUDED.license_url, + allowed_purposes = EXCLUDED.allowed_purposes, + pii_allowed = EXCLUDED.pii_allowed, + redistribution = EXCLUDED.redistribution, + attribution = EXCLUDED.attribution, + verified_at = EXCLUDED.verified_at, + verified_by = EXCLUDED.verified_by, + notes = EXCLUDED.notes, + updated_at = now()`; + } + + async getRights(sourceId: string): Promise { + const rows = await this.sql` + SELECT source_id, tenant_id, license_id, license_url, allowed_purposes, pii_allowed, redistribution, attribution, verified_at, verified_by, notes + FROM opt_source_rights WHERE source_id = ${sourceId} LIMIT 1`; + return rows[0] ? toRights(rows[0]) : undefined; + } + + async setProvenance(prov: SourceProvenance): Promise { + await this.sql` + INSERT INTO opt_source_provenance + (source_id, tenant_id, method, authority, compliance_evidence_ref, cadence, freshness_sla_hours, updated_at) + VALUES + (${prov.source_id}, ${prov.tenant_id}, ${prov.method}, ${prov.authority ?? null}, + ${prov.compliance_evidence_ref ?? null}, ${prov.cadence ?? null}, ${prov.freshness_sla_hours ?? null}, ${prov.updated_at}) + ON CONFLICT (source_id) DO UPDATE SET + tenant_id = EXCLUDED.tenant_id, + method = EXCLUDED.method, + authority = EXCLUDED.authority, + compliance_evidence_ref = EXCLUDED.compliance_evidence_ref, + cadence = EXCLUDED.cadence, + freshness_sla_hours = EXCLUDED.freshness_sla_hours, + updated_at = EXCLUDED.updated_at`; + } + + async getProvenance(sourceId: string): Promise { + const rows = await this.sql` + SELECT source_id, tenant_id, method, authority, compliance_evidence_ref, cadence, freshness_sla_hours, updated_at + FROM opt_source_provenance WHERE source_id = ${sourceId} LIMIT 1`; + return rows[0] ? toProvenance(rows[0]) : undefined; + } + + async recordSuccess(sourceId: string, tenantId: string, atIso: string): Promise { + await this.sql` + INSERT INTO opt_source_freshness (source_id, tenant_id, last_success_at) + VALUES (${sourceId}, ${tenantId}, ${atIso}) + ON CONFLICT (source_id) DO UPDATE SET + last_success_at = EXCLUDED.last_success_at, + tenant_id = EXCLUDED.tenant_id`; + } + + async getLastSuccess(sourceId: string): Promise { + const rows = await this.sql<{ last_success_at: string | Date }[]>` + SELECT last_success_at FROM opt_source_freshness WHERE source_id = ${sourceId} LIMIT 1`; + return rows[0] ? toIsoString(rows[0].last_success_at) : undefined; + } + + async appendHealthCheck(check: SourceHealthCheck): Promise { + await this.sql` + INSERT INTO opt_source_health_checks (source_id, tenant_id, status, checked_at, latency_ms, detail) + VALUES (${check.source_id}, ${check.tenant_id}, ${check.status}, ${check.checked_at}, ${check.latency_ms ?? null}, ${check.detail ?? null})`; + // Bounded history -- keep only the latest 100 observations per source + // (health is an operational signal, not an audit log; AUD0 owns history), + // same bound InMemorySourceRegistryRepository enforces on write. + await this.sql` + DELETE FROM opt_source_health_checks + WHERE source_id = ${check.source_id} + AND health_check_id NOT IN ( + SELECT health_check_id FROM opt_source_health_checks + WHERE source_id = ${check.source_id} + ORDER BY checked_at DESC, health_check_id DESC + LIMIT 100 + )`; + } + + async listHealthChecks(sourceId: string, limit = 100): Promise { + const rows = await this.sql` + SELECT source_id, tenant_id, status, checked_at, latency_ms, detail + FROM opt_source_health_checks + WHERE source_id = ${sourceId} + ORDER BY checked_at DESC, health_check_id DESC + LIMIT ${limit}`; + return rows.map(toHealthCheck); + } +} diff --git a/src/repositories/postgres-source-repository.ts b/src/repositories/postgres-source-repository.ts new file mode 100644 index 0000000..970e911 --- /dev/null +++ b/src/repositories/postgres-source-repository.ts @@ -0,0 +1,98 @@ +/** + * src/repositories/postgres-source-repository.ts + * ================================================= + * Real Postgres backing for SourceRepository, against `opt_sources` + * (sql/002_source_registry.sql). Same validation/error semantics as + * InMemorySourceRepository ("already exists", "name is required", "not + * found" -- exact message text preserved so every existing caller/test that + * matches on those strings keeps working unmodified). + */ +import type { CreateSourceInput, Source, SourceStatus, SourceType } from "../types/source"; +import type { RepositorySql } from "./db"; +import type { SourceListFilter, SourceRepository } from "./source-repository"; + +interface SourceRow { + source_id: string; + tenant_id: string; + project_id: string; + source_type: string; + name: string; + adapter_id: string | null; + access: string | null; + description: string | null; + status: string; + correlation_id: string | null; + created_at: string | Date; + updated_at: string | Date | null; + created_by: string | null; +} + +function toIsoString(v: string | Date): string { + return v instanceof Date ? v.toISOString() : v; +} + +function toSource(r: SourceRow): Source { + return { + source_id: r.source_id, + tenant_id: r.tenant_id, + project_id: r.project_id, + source_type: r.source_type as SourceType, + name: r.name, + adapter_id: r.adapter_id ?? undefined, + access: (r.access as Source["access"]) ?? undefined, + description: r.description ?? undefined, + status: r.status as SourceStatus, + correlation_id: r.correlation_id ?? undefined, + created_at: toIsoString(r.created_at), + updated_at: r.updated_at ? toIsoString(r.updated_at) : undefined, + created_by: r.created_by ?? undefined, + }; +} + +export class PostgresSourceRepository implements SourceRepository { + constructor(private readonly sql: RepositorySql) {} + + async create(input: CreateSourceInput): Promise { + if (!input.name) throw new Error("name is required"); + + const existing = await this.sql<{ source_id: string }[]>` + SELECT source_id FROM opt_sources WHERE source_id = ${input.source_id} LIMIT 1`; + if (existing.length > 0) { + throw new Error(`Source ${input.source_id} already exists`); + } + + const rows = await this.sql` + INSERT INTO opt_sources + (source_id, tenant_id, project_id, source_type, name, adapter_id, access, description, status, correlation_id, created_by) + VALUES + (${input.source_id}, ${input.tenant_id}, ${input.project_id}, ${input.source_type}, ${input.name}, + ${input.adapter_id ?? null}, ${input.access ?? null}, ${input.description ?? null}, + ${input.status ?? "active"}, ${input.correlation_id ?? null}, ${input.created_by ?? null}) + RETURNING source_id, tenant_id, project_id, source_type, name, adapter_id, access, description, + status, correlation_id, created_at, updated_at, created_by`; + return toSource(rows[0]); + } + + async get(sourceId: string): Promise { + const rows = await this.sql` + SELECT source_id, tenant_id, project_id, source_type, name, adapter_id, access, description, + status, correlation_id, created_at, updated_at, created_by + FROM opt_sources WHERE source_id = ${sourceId} LIMIT 1`; + if (!rows[0]) throw new Error(`Source ${sourceId} not found`); + return toSource(rows[0]); + } + + async list(filter?: SourceListFilter): Promise { + const rows = await this.sql` + SELECT source_id, tenant_id, project_id, source_type, name, adapter_id, access, description, + status, correlation_id, created_at, updated_at, created_by + FROM opt_sources + WHERE 1=1 + ${filter?.tenant_id ? this.sql`AND tenant_id = ${filter.tenant_id}` : this.sql``} + ${filter?.project_id ? this.sql`AND project_id = ${filter.project_id}` : this.sql``} + ${filter?.source_type ? this.sql`AND source_type = ${filter.source_type}` : this.sql``} + ${filter?.status ? this.sql`AND status = ${filter.status}` : this.sql``} + ORDER BY created_at ASC`; + return rows.map(toSource); + } +} diff --git a/src/repositories/source-registry-repository.ts b/src/repositories/source-registry-repository.ts new file mode 100644 index 0000000..4560e26 --- /dev/null +++ b/src/repositories/source-registry-repository.ts @@ -0,0 +1,45 @@ +/** + * src/repositories/source-registry-repository.ts + * ================================================= + * Storage seam for the four Maps src/services/source-registry-service.ts + * used to hold rights, provenance, health checks, and last-success + * timestamps. Same split as source-repository.ts: one interface, an + * in-memory implementation (extracted verbatim) and a Postgres + * implementation (sql/002_source_registry.sql's `opt_source_rights`, + * `opt_source_provenance`, `opt_source_health_checks`, `opt_source_freshness` + * tables), chosen once via `resolveStoreMode`. + * + * Tenant isolation is NOT this layer's job -- exactly as before, when these + * were bare Maps keyed only by source_id. SourceRegistryService's + * `assertSourceTenant` gate is what enforces the cross-tenant NOT_FOUND + * posture (§50), and it runs before every call into this repository. This + * layer trusts the caller, same as the Maps did. + */ +import { resolveStoreMode } from "../platform/dependency-mode"; +import type { SourceHealthCheck, SourceProvenance, SourceRights } from "../types/source-registry"; +import { getRepositorySql, repositoryDatabaseUrl } from "./db"; +import { InMemorySourceRegistryRepository } from "./in-memory-source-registry-repository"; +import { PostgresSourceRegistryRepository } from "./postgres-source-registry-repository"; + +export interface SourceRegistryRepository { + setRights(rights: SourceRights): Promise; + getRights(sourceId: string): Promise; + + setProvenance(prov: SourceProvenance): Promise; + getProvenance(sourceId: string): Promise; + + recordSuccess(sourceId: string, tenantId: string, atIso: string): Promise; + getLastSuccess(sourceId: string): Promise; + + appendHealthCheck(check: SourceHealthCheck): Promise; + /** Newest-first, bounded to the latest `limit` observations (default 100) -- health is an operational signal, not an audit log; AUD0 owns history. */ + listHealthChecks(sourceId: string, limit?: number): Promise; +} + +export function createSourceRegistryRepository(databaseUrl?: string): SourceRegistryRepository { + const mode = resolveStoreMode("0pt.source_registry.metadata", databaseUrl ?? repositoryDatabaseUrl()); + if (mode.mode === "postgres") { + return new PostgresSourceRegistryRepository(getRepositorySql(mode.url)); + } + return new InMemorySourceRegistryRepository(); +} diff --git a/src/repositories/source-repository.ts b/src/repositories/source-repository.ts new file mode 100644 index 0000000..f00e1cf --- /dev/null +++ b/src/repositories/source-repository.ts @@ -0,0 +1,44 @@ +/** + * src/repositories/source-repository.ts + * ======================================= + * Storage seam for src/services/source-service.ts (0pt.source.schema.v1). + * Extracted from what used to be a bare `new Map()` inside SourceService + * itself so the same business logic (validation, "already exists"/"not + * found" semantics, filtering) can run against either an in-memory Map + * (tests, local dev, non-production with no DATABASE_URL) or real Postgres + * (sql/002_source_registry.sql's `opt_sources` table) without the service + * layer knowing which. + * + * `createSourceRepository` is the single choice point, delegating to + * `resolveStoreMode` (src/platform/dependency-mode.ts) -- same + * production-safety contract as every other P0 store: no DATABASE_URL in a + * production runtime throws DEPENDENCY_UNAVAILABLE at construction rather + * than silently serving the in-memory fallback. + */ +import { resolveStoreMode } from "../platform/dependency-mode"; +import type { CreateSourceInput, Source, SourceStatus, SourceType } from "../types/source"; +import { getRepositorySql, repositoryDatabaseUrl } from "./db"; +import { InMemorySourceRepository } from "./in-memory-source-repository"; +import { PostgresSourceRepository } from "./postgres-source-repository"; + +export interface SourceListFilter { + tenant_id?: string; + project_id?: string; + source_type?: SourceType; + status?: SourceStatus; +} + +export interface SourceRepository { + create(input: CreateSourceInput): Promise; + /** Throws (message contains "not found") when no such source exists -- matches the pre-migration Map behavior every caller already handles. */ + get(sourceId: string): Promise; + list(filter?: SourceListFilter): Promise; +} + +export function createSourceRepository(databaseUrl?: string): SourceRepository { + const mode = resolveStoreMode("0pt.source_registry.sources", databaseUrl ?? repositoryDatabaseUrl()); + if (mode.mode === "postgres") { + return new PostgresSourceRepository(getRepositorySql(mode.url)); + } + return new InMemorySourceRepository(); +} diff --git a/src/services/source-rate-policy-service.ts b/src/services/source-rate-policy-service.ts index 088a1bd..2e53bab 100644 --- a/src/services/source-rate-policy-service.ts +++ b/src/services/source-rate-policy-service.ts @@ -1,4 +1,8 @@ -import type { SourceRatePolicy, SourceRatePolicyStatus, CreateSourceRatePolicyInput } from "../types/source-rate-policy"; +import type { + SourceRatePolicy, + SourceRatePolicyStatus, + CreateSourceRatePolicyInput, +} from "../types/source-rate-policy"; import { createAud0Client, type Aud0Client } from "../clients/aud0-client"; import { SourceService } from "./source-service"; @@ -33,7 +37,7 @@ export class SourceRatePolicyService { constructor(opts?: { aud0BaseUrl?: string; sourceService?: SourceService }) { this.aud0 = createAud0Client(opts?.aud0BaseUrl); - this.sourceService = opts?.sourceService ?? new SourceService(opts?.aud0BaseUrl); + this.sourceService = opts?.sourceService ?? new SourceService({ aud0BaseUrl: opts?.aud0BaseUrl }); } async createSourceRatePolicy(input: CreateSourceRatePolicyInput): Promise { diff --git a/src/services/source-registry-service.ts b/src/services/source-registry-service.ts index d3ad36f..e9ec51d 100644 --- a/src/services/source-registry-service.ts +++ b/src/services/source-registry-service.ts @@ -14,20 +14,33 @@ * Tenant isolation: every read path REQUIRES a tenant_id and filters on it — * a source registered by tenant A is invisible to tenant B (tested in * tests/p0/source-registry.test.ts, incl. the cross-tenant attack case §50). + * `getHealth`/`getFreshness` now take tenant_id directly and run the same + * `assertSourceTenant` gate every other accessor does — they are exposed as + * standalone HTTP routes now (app/api/0pt/sources/[sourceId]/health, + * .../freshness), so they can no longer rely on an earlier caller having + * already checked tenancy on their behalf. * * Purpose enforcement: `assertPurposeAllowed` is the single hook other 0pt * capabilities call before using a source for a given purpose (SearchService * and DatasetService both do). No rights record ⇒ fail closed (deny). * - * Persistence: in-memory Maps behind the same "pure core, thin registry" - * shape as the sibling P0 services. Production boot is protected by - * `resolveStoreMode` in the composition root (app/api routes construct with - * the shared registry): with no database configured a production runtime - * throws DEPENDENCY_UNAVAILABLE instead of serving Maps (Master-DoD §3). + * Persistence: rights/provenance/health/freshness now live behind + * SourceRegistryRepository (src/repositories/source-registry-repository.ts) + * — real Postgres (`opt_source_rights`/`opt_source_provenance`/ + * `opt_source_health_checks`/`opt_source_freshness`, + * sql/002_source_registry.sql) when DATABASE_URL is configured, in-memory + * Maps otherwise. `resolveStoreMode` still fails production boot hard with + * no DATABASE_URL (Master-DoD §3) — the difference from before is that a + * configured production now actually WORKS (opt_sources/etc. exist), not + * just fails safely. */ import { createAud0Client, type Aud0Client } from "../clients/aud0-client"; import { SiaxError } from "../platform/errors"; -import { SourceService } from "./source-service"; +import { + createSourceRegistryRepository, + type SourceRegistryRepository, +} from "../repositories/source-registry-repository"; +import { SourceService, type SourceServiceOptions } from "./source-service"; import { SourceRatePolicyService } from "./source-rate-policy-service"; import type { CreateSourceInput, Source } from "../types/source"; import type { @@ -46,21 +59,27 @@ export interface RegisterSourceInput extends CreateSourceInput { } export class SourceRegistryService { - private readonly rights = new Map(); - private readonly provenance = new Map(); - private readonly healthChecks = new Map(); - private readonly lastSuccess = new Map(); + private readonly repo: SourceRegistryRepository; private readonly aud0: Aud0Client; readonly sources: SourceService; readonly ratePolicies: SourceRatePolicyService; constructor(opts?: { aud0BaseUrl?: string; + /** Explicit DATABASE_URL override — mainly for tests. Falls back to resolveStoreMode's own env lookup when omitted. */ + databaseUrl?: string; sourceService?: SourceService; ratePolicyService?: SourceRatePolicyService; + /** Inject a repository directly (tests) — bypasses resolveStoreMode entirely. */ + repository?: SourceRegistryRepository; }) { this.aud0 = createAud0Client(opts?.aud0BaseUrl); - this.sources = opts?.sourceService ?? new SourceService(opts?.aud0BaseUrl); + this.repo = opts?.repository ?? createSourceRegistryRepository(opts?.databaseUrl); + const sourceServiceOpts: SourceServiceOptions = { + aud0BaseUrl: opts?.aud0BaseUrl, + databaseUrl: opts?.databaseUrl, + }; + this.sources = opts?.sourceService ?? new SourceService(sourceServiceOpts); this.ratePolicies = opts?.ratePolicyService ?? new SourceRatePolicyService({ aud0BaseUrl: opts?.aud0BaseUrl, sourceService: this.sources }); @@ -91,7 +110,7 @@ export class SourceRegistryService { // ------------------------------------------------------------------------- async setRights(rights: SourceRights): Promise { await this.assertSourceTenant(rights.source_id, rights.tenant_id); - this.rights.set(rights.source_id, rights); + await this.repo.setRights(rights); await this.emit("0pt.source_rights_set", rights.tenant_id, rights.source_id, { license_id: rights.license_id, allowed_purposes: rights.allowed_purposes, @@ -102,7 +121,7 @@ export class SourceRegistryService { async getRights(sourceId: string, tenantId: string): Promise { await this.assertSourceTenant(sourceId, tenantId); - return this.rights.get(sourceId); + return this.repo.getRights(sourceId); } /** @@ -112,19 +131,19 @@ export class SourceRegistryService { */ async assertPurposeAllowed(sourceId: string, tenantId: string, purpose: string): Promise { await this.assertSourceTenant(sourceId, tenantId); - const rights = this.rights.get(sourceId); + const rights = await this.repo.getRights(sourceId); if (!rights) { throw new SiaxError( "AUTHORIZATION_DENIED", "0pt.source_registry", - `source ${sourceId} has no rights record — purpose "${purpose}" denied (fail closed)` + `source ${sourceId} has no rights record — purpose "${purpose}" denied (fail closed)`, ); } if (!rights.allowed_purposes.includes(purpose)) { throw new SiaxError( "AUTHORIZATION_DENIED", "0pt.source_registry", - `purpose "${purpose}" is not in source ${sourceId}'s allowed purposes` + `purpose "${purpose}" is not in source ${sourceId}'s allowed purposes`, ); } return rights; @@ -135,7 +154,7 @@ export class SourceRegistryService { // ------------------------------------------------------------------------- async setProvenance(prov: SourceProvenance): Promise { await this.assertSourceTenant(prov.source_id, prov.tenant_id); - this.provenance.set(prov.source_id, prov); + await this.repo.setProvenance(prov); await this.emit("0pt.source_provenance_set", prov.tenant_id, prov.source_id, { method: prov.method, authority: prov.authority, @@ -146,18 +165,19 @@ export class SourceRegistryService { async getProvenance(sourceId: string, tenantId: string): Promise { await this.assertSourceTenant(sourceId, tenantId); - return this.provenance.get(sourceId); + return this.repo.getProvenance(sourceId); } /** Record a successful data observation (drives freshness). */ async recordSuccess(sourceId: string, tenantId: string, atIso?: string): Promise { await this.assertSourceTenant(sourceId, tenantId); - this.lastSuccess.set(sourceId, atIso ?? new Date().toISOString()); + await this.repo.recordSuccess(sourceId, tenantId, atIso ?? new Date().toISOString()); } - getFreshness(sourceId: string, now: Date = new Date()): SourceFreshness { - const prov = this.provenance.get(sourceId); - const last = this.lastSuccess.get(sourceId); + async getFreshness(sourceId: string, tenantId: string, now: Date = new Date()): Promise { + await this.assertSourceTenant(sourceId, tenantId); + const prov = await this.repo.getProvenance(sourceId); + const last = await this.repo.getLastSuccess(sourceId); const sla = prov?.freshness_sla_hours; if (!last) { return { source_id: sourceId, freshness_sla_hours: sla, fresh: sla === undefined }; @@ -177,24 +197,21 @@ export class SourceRegistryService { // ------------------------------------------------------------------------- async recordHealthCheck(check: SourceHealthCheck): Promise { await this.assertSourceTenant(check.source_id, check.tenant_id); - const list = this.healthChecks.get(check.source_id) ?? []; - list.push(check); - // Bounded history — health is an operational signal, not an audit log - // (AUD0 owns history); keep the latest 100 observations. - if (list.length > 100) list.splice(0, list.length - 100); - this.healthChecks.set(check.source_id, list); + await this.repo.appendHealthCheck(check); if (check.status === "up") { await this.recordSuccess(check.source_id, check.tenant_id, check.checked_at); } - return this.getHealth(check.source_id); + return this.getHealth(check.source_id, check.tenant_id); } - getHealth(sourceId: string): SourceHealth { - const list = this.healthChecks.get(sourceId) ?? []; - const last = list[list.length - 1]; + async getHealth(sourceId: string, tenantId: string): Promise { + await this.assertSourceTenant(sourceId, tenantId); + // Newest-first (repository contract) — index 0 is the most recent check. + const list = await this.repo.listHealthChecks(sourceId); + const last = list[0]; let consecutiveFailures = 0; - for (let i = list.length - 1; i >= 0; i--) { - if (list[i].status === "down" || list[i].status === "degraded") consecutiveFailures++; + for (const check of list) { + if (check.status === "down" || check.status === "degraded") consecutiveFailures++; else break; } const status: SourceHealthStatus = last?.status ?? "unknown"; @@ -223,27 +240,41 @@ export class SourceRegistryService { source_type: filter.source_type, status: filter.status, }); - if (filter.purpose !== undefined) { - results = results.filter((s) => { - const rights = this.rights.get(s.source_id); - return rights !== undefined && rights.allowed_purposes.includes(filter.purpose as string); - }); - } - if (filter.redistribution !== undefined) { - results = results.filter((s) => this.rights.get(s.source_id)?.redistribution === filter.redistribution); + if (filter.purpose !== undefined || filter.redistribution !== undefined) { + const rightsEntries = await Promise.all( + results.map(async (s) => [s.source_id, await this.repo.getRights(s.source_id)] as const), + ); + const rightsBySource = new Map(rightsEntries); + if (filter.purpose !== undefined) { + results = results.filter((s) => { + const rights = rightsBySource.get(s.source_id); + return rights !== undefined && rights.allowed_purposes.includes(filter.purpose as string); + }); + } + if (filter.redistribution !== undefined) { + results = results.filter( + (s) => rightsBySource.get(s.source_id)?.redistribution === filter.redistribution, + ); + } } return results; } async describeSource(sourceId: string, tenantId: string): Promise { const source = await this.assertSourceTenant(sourceId, tenantId); - const policies = await this.ratePolicies.listSourceRatePolicies({ tenant_id: tenantId, source_id: sourceId }); + const [policies, rights, provenance, health, freshness] = await Promise.all([ + this.ratePolicies.listSourceRatePolicies({ tenant_id: tenantId, source_id: sourceId }), + this.repo.getRights(sourceId), + this.repo.getProvenance(sourceId), + this.getHealth(sourceId, tenantId), + this.getFreshness(sourceId, tenantId), + ]); return { source, - rights: this.rights.get(sourceId), - provenance: this.provenance.get(sourceId), - health: this.getHealth(sourceId), - freshness: this.getFreshness(sourceId), + rights, + provenance, + health, + freshness, rate_policy: policies[0], }; } @@ -276,7 +307,7 @@ export class SourceRegistryService { eventType: string, tenantId: string, sourceId: string, - payload: Record + payload: Record, ): Promise { await this.aud0.ingestEvent({ event_id: `evt_${eventType}_${sourceId}_${Date.now()}`, diff --git a/src/services/source-registry.ts b/src/services/source-registry.ts new file mode 100644 index 0000000..50121ff --- /dev/null +++ b/src/services/source-registry.ts @@ -0,0 +1,30 @@ +/** + * src/services/source-registry.ts + * ================================= + * Shared singleton instances for the source-registry route group + * (app/api/0pt/sources/*), matching src/services/lakehouse-mapping-registry.ts's + * precedent. + * + * BEFORE_STATE.md §1 flagged this as materially worse than plain + * in-memory-per-module: each route file previously constructed its own + * `new SourceService()` (and `new SourceRatePolicyService()`), so a source + * registered via one route module was invisible to another within the same + * running process — "not even a shared process-wide singleton". Once + * DATABASE_URL is configured this stops mattering (state lives in Postgres, + * genuinely shared across processes/deploys — see src/repositories/*), but + * it still matters for the in-memory (no DATABASE_URL) dev/test mode, where + * state must be shared within one process to be usable at all. Constructing + * all three here once, wired together, and importing them everywhere fixes + * both call sites this repo already had (sources/rate-policies/*, + * sources/[sourceId]/query) plus every new route this pass adds. + */ +import { SourceService } from "./source-service"; +import { SourceRatePolicyService } from "./source-rate-policy-service"; +import { SourceRegistryService } from "./source-registry-service"; + +export const sourceService = new SourceService(); +export const sourceRatePolicyService = new SourceRatePolicyService({ sourceService }); +export const sourceRegistryService = new SourceRegistryService({ + sourceService, + ratePolicyService: sourceRatePolicyService, +}); diff --git a/src/services/source-service.ts b/src/services/source-service.ts index 51df9d7..5eade69 100644 --- a/src/services/source-service.ts +++ b/src/services/source-service.ts @@ -7,107 +7,74 @@ * and `SourceRatePolicyService` (src/services/source-rate-policy-service.ts) * both look a `source_id` up against — see docs/P0_QUERY_SOURCE_FLOW.md. * - * Deliberately small and in-memory, same "pure core, thin registry" shape - * every other P0 service in this repo already uses; a real deployment backs - * this with `opt_sources`. In-memory scope: each `new SourceService()` owns - * its own registry — the pre-existing limitation docs/P0_QUERY_SOURCE_FLOW.md's - * "Known gaps" section documents (a Source registered via one API route module - * is not yet visible to another route module in the same running process). - * Callers that need a shared view pass an existing instance in via the - * `sourceService` constructor option, exactly like SourceQueryService and - * SourceRatePolicyService already do. + * Persistence: storage now lives behind SourceRepository + * (src/repositories/source-repository.ts) — real Postgres (`opt_sources`, + * sql/002_source_registry.sql) when DATABASE_URL is configured, an + * in-memory Map otherwise (non-production only; resolveStoreMode fails hard + * in production with no URL). This service still owns the business rules + * (validation, tenant-agnostic CRUD orchestration) and the AUD0 emission — + * only the storage moved. * - * Registering a source emits a best-effort AUD0 `0pt.source_created` event — - * the same "no mutation without an evented trail" doctrine every other P0 - * registry service follows. With no AUD0_URL configured that emission is a - * no-op (see aud0-client.ts), so this stays hermetic in tests and local dev. + * Callers that need a shared view across route modules should use the + * shared singleton in src/services/source-registry.ts rather than + * constructing a fresh `new SourceService()` per module — see that file's + * doc comment for the "each route owned its own registry" history this + * fixes. */ import { createAud0Client, type Aud0Client } from "../clients/aud0-client"; +import { + createSourceRepository, + type SourceRepository, + type SourceListFilter, +} from "../repositories/source-repository"; import type { CreateSourceInput, Source, SourceStatus, SourceType } from "../types/source"; -/** - * Storage shape: nullable columns, exactly as the eventual `opt_sources` table - * would hold them. `toSource()` converts back to the `Source` DTO (undefined - * rather than null) so callers never have to handle both. - */ -interface SourceRecord { - source_id: string; - tenant_id: string; - project_id: string; - source_type: string; - name: string; - adapter_id: string | null; - access: string | null; - description: string | null; - status: string; - correlation_id: string | null; - created_at: string; - updated_at: string | null; - created_by: string | null; +export interface SourceServiceOptions { + aud0BaseUrl?: string; + /** Explicit DATABASE_URL override — mainly for tests pointing at a specific Postgres instance. Falls back to resolveStoreMode's own env lookup when omitted. */ + databaseUrl?: string; + /** Inject a repository directly (tests) — bypasses resolveStoreMode entirely. */ + repository?: SourceRepository; } export class SourceService { - private sources: Map = new Map(); - private aud0: Aud0Client; + private readonly repo: SourceRepository; + private readonly aud0: Aud0Client; - constructor(aud0BaseUrl?: string) { - this.aud0 = createAud0Client(aud0BaseUrl); + constructor(opts?: SourceServiceOptions) { + this.aud0 = createAud0Client(opts?.aud0BaseUrl); + this.repo = opts?.repository ?? createSourceRepository(opts?.databaseUrl); } async createSource(input: CreateSourceInput): Promise { - if (this.sources.has(input.source_id)) { - throw new Error(`Source ${input.source_id} already exists`); - } - if (!input.name) { - throw new Error("name is required"); - } - - const now = new Date().toISOString(); - const record: SourceRecord = { - source_id: input.source_id, - tenant_id: input.tenant_id, - project_id: input.project_id, - source_type: input.source_type, - name: input.name, - adapter_id: input.adapter_id ?? null, - access: input.access ?? null, - description: input.description ?? null, - status: input.status ?? "active", - correlation_id: input.correlation_id ?? null, - created_at: now, - updated_at: null, - created_by: input.created_by ?? null, - }; - this.sources.set(record.source_id, record); + const source = await this.repo.create(input); await this.aud0.ingestEvent({ - event_id: `evt_0pt_source_created_${record.source_id}`, + event_id: `evt_0pt_source_created_${source.source_id}`, event_type: "0pt.source_created", - tenant_id: record.tenant_id, - project_id: record.project_id, - actor_id: record.created_by ?? "system", + tenant_id: source.tenant_id, + project_id: source.project_id, + actor_id: source.created_by ?? "system", app_id: "0pt", capability_id: "0pt.create_source", resource_type: "source", - resource_id: record.source_id, + resource_id: source.source_id, risk_level: "L2", payload: { - source_type: record.source_type, - name: record.name, - adapter_id: record.adapter_id, - access: record.access, + source_type: source.source_type, + name: source.name, + adapter_id: source.adapter_id, + access: source.access, }, - created_at: now, - correlation_id: record.correlation_id ?? undefined, + created_at: source.created_at, + correlation_id: source.correlation_id, }); - return this.toSource(record); + return source; } async getSource(sourceId: string): Promise { - const record = this.sources.get(sourceId); - if (!record) throw new Error(`Source ${sourceId} not found`); - return this.toSource(record); + return this.repo.get(sourceId); } async listSources(filter?: { @@ -116,29 +83,6 @@ export class SourceService { source_type?: SourceType; status?: SourceStatus; }): Promise { - let results = Array.from(this.sources.values()); - if (filter?.tenant_id) results = results.filter((r) => r.tenant_id === filter.tenant_id); - if (filter?.project_id) results = results.filter((r) => r.project_id === filter.project_id); - if (filter?.source_type) results = results.filter((r) => r.source_type === filter.source_type); - if (filter?.status) results = results.filter((r) => r.status === filter.status); - return results.map((r) => this.toSource(r)); - } - - private toSource(r: SourceRecord): Source { - return { - source_id: r.source_id, - tenant_id: r.tenant_id, - project_id: r.project_id, - source_type: r.source_type as SourceType, - name: r.name, - adapter_id: r.adapter_id ?? undefined, - access: (r.access as Source["access"]) ?? undefined, - description: r.description ?? undefined, - status: r.status as SourceStatus, - correlation_id: r.correlation_id ?? undefined, - created_at: r.created_at, - updated_at: r.updated_at ?? undefined, - created_by: r.created_by ?? undefined, - }; + return this.repo.list(filter as SourceListFilter | undefined); } } diff --git a/src/validation/0pt-schemas.ts b/src/validation/0pt-schemas.ts index 5578681..8231700 100644 --- a/src/validation/0pt-schemas.ts +++ b/src/validation/0pt-schemas.ts @@ -95,6 +95,64 @@ export const createSourceRatePolicySchema = z.object({ created_by: z.string().min(1).optional(), }); +/** Nested `rights` block on POST /api/0pt/sources, and the body of PUT /api/0pt/sources/[sourceId]/rights -- mirrors SourceRights (minus source_id/tenant_id, which come from the URL/outer body). */ +const sourceRightsInputSchema = z.object({ + license_id: z.string().min(1), + license_url: z.string().min(1).optional(), + allowed_purposes: z.array(z.string().min(1)).default([]), + pii_allowed: z.boolean(), + redistribution: z.enum(['internal', 'customer_deliverable', 'public', 'prohibited']), + attribution: z.string().min(1).optional(), + verified_at: z.string().min(1), + verified_by: z.string().min(1), + notes: z.string().min(1).optional(), +}); + +/** Nested `provenance` block on POST /api/0pt/sources, and the body of PUT /api/0pt/sources/[sourceId]/provenance -- mirrors SourceProvenance (minus source_id/tenant_id/updated_at, which are set by the service). */ +const sourceProvenanceInputSchema = z.object({ + method: z.enum(['official_api', 'bulk_download', 'authorized_web', 'licensed_feed', 'manual_upload']), + authority: z.string().min(1).optional(), + compliance_evidence_ref: z.string().min(1).optional(), + cadence: z.string().min(1).optional(), + freshness_sla_hours: z.number().nonnegative().optional(), +}); + +/** POST /api/0pt/sources -- source.register. */ +export const registerSourceSchema = z.object({ + source_id: z.string().min(1), + tenant_id: z.string().min(1), + project_id: z.string().min(1), + source_type: z.enum(['database', 'file', 'web', 'api', 'stream', 'manual']), + name: z.string().min(1), + adapter_id: z.string().min(1).optional(), + access: z.enum(['api', 'bulk', 'authorized_web', 'licensed']).optional(), + description: z.string().min(1).optional(), + status: z.enum(['active', 'paused', 'archived', 'deprecated']).optional(), + correlation_id: z.string().min(1).optional(), + created_by: z.string().min(1).optional(), + rights: sourceRightsInputSchema.optional(), + provenance: sourceProvenanceInputSchema.optional(), +}); + +/** PUT /api/0pt/sources/[sourceId]/rights -- source.set_rights. */ +export const setSourceRightsSchema = sourceRightsInputSchema.extend({ + tenant_id: z.string().min(1), +}); + +/** PUT /api/0pt/sources/[sourceId]/provenance -- source.set_provenance. */ +export const setSourceProvenanceSchema = sourceProvenanceInputSchema.extend({ + tenant_id: z.string().min(1), +}); + +/** POST /api/0pt/sources/[sourceId]/health -- source.health (record a health check). */ +export const recordSourceHealthCheckSchema = z.object({ + tenant_id: z.string().min(1), + status: z.enum(['up', 'degraded', 'down', 'unknown']), + checked_at: z.string().min(1).optional(), + latency_ms: z.number().nonnegative().optional(), + detail: z.string().min(1).optional(), +}); + /** POST /api/0pt/lakehouse/mappings -- mirrors MapDatasetToIcebergTableInput. */ export const mapDatasetToIcebergTableSchema = z.object({ tenant_id: z.string().min(1), diff --git a/tests/p0/source-registry-postgres.realinfra.test.ts b/tests/p0/source-registry-postgres.realinfra.test.ts new file mode 100644 index 0000000..775bdd5 --- /dev/null +++ b/tests/p0/source-registry-postgres.realinfra.test.ts @@ -0,0 +1,340 @@ +/** + * REAL Postgres integration test (Master-DoD §48/§50) for the Source + * Registry's new persistence layer (src/repositories/*, + * sql/002_source_registry.sql) -- the P010 task's central deliverable: this + * is what proves the Map -> Postgres migration actually works, not just + * that it typechecks. + * + * Same honest-degrade convention as tests/p0/iceberg-write-service.realinfra.test.ts + * and tests/integration/analytics-clickhouse.integration.test.ts: gated on + * an env var (SOURCE_REGISTRY_TEST_DATABASE_URL), SKIPS HONESTLY when unset, + * never fabricates success. Point it at a disposable Postgres, e.g.: + * + * docker run -d --name opt-source-registry-pgtest \ + * -e POSTGRES_USER=opt -e POSTGRES_PASSWORD=opt_test_pw -e POSTGRES_DB=opt_test \ + * -p 15433:5432 postgres:16-alpine + * SOURCE_REGISTRY_TEST_DATABASE_URL=postgres://opt:opt_test_pw@localhost:15433/opt_test \ + * pnpm vitest run tests/p0/source-registry-postgres.realinfra.test.ts + * + * This is NOT a mock-repository test -- every assertion below is driven + * through SourceRegistryService with `databaseUrl` set, i.e. the real + * PostgresSourceRepository/PostgresSourceRegistryRepository HTTP-free SQL + * path (postgres.js against the live container), with a handful of + * assertions cross-checked with direct SQL against the same database to + * prove the rows are really there, not an artifact of the service's own + * in-process bookkeeping. + * + * beforeAll applies sql/002_source_registry.sql itself (every statement is + * `CREATE TABLE/INDEX IF NOT EXISTS`, so this is idempotent and safe to run + * against an already-migrated database too) -- this test does not require + * the operator to have pre-run migrations. + */ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import fs from "node:fs"; +import path from "node:path"; +import postgres from "postgres"; +import { SourceRegistryService } from "@/src/services/source-registry-service"; +import { closeRepositorySqlForTests } from "@/src/repositories/db"; + +const TEST_DATABASE_URL = process.env.SOURCE_REGISTRY_TEST_DATABASE_URL; +const run = TEST_DATABASE_URL ? describe : describe.skip; + +let adminSql: ReturnType | undefined; +let reachable = false; +let unreachableReason = ""; + +beforeAll(async () => { + if (!TEST_DATABASE_URL) { + console.warn( + "[realinfra] SOURCE_REGISTRY_TEST_DATABASE_URL not set this run -- skipping honestly, not fabricating success.", + ); + return; + } + try { + adminSql = postgres(TEST_DATABASE_URL, { prepare: false, ssl: false, max: 2 }); + await Promise.race([ + adminSql`SELECT 1`, + new Promise((_, reject) => setTimeout(() => reject(new Error("connect timeout")), 5000)), + ]); + const migrationPath = path.join(process.cwd(), "sql", "002_source_registry.sql"); + const migration = fs.readFileSync(migrationPath, "utf8"); + await adminSql.unsafe(migration); + reachable = true; + } catch (err) { + reachable = false; + unreachableReason = err instanceof Error ? err.message : String(err); + } +}, 30000); + +afterAll(async () => { + await adminSql?.end({ timeout: 5 }); + await closeRepositorySqlForTests(); +}); + +function svc(): SourceRegistryService { + return new SourceRegistryService({ databaseUrl: TEST_DATABASE_URL }); +} + +run("SourceRegistryService against a LIVE Postgres (sql/002_source_registry.sql)", () => { + it("persists across independent service instances -- the exact bug this migration fixes (in-memory Maps were per-instance, not even per-process)", async (ctx) => { + if (!reachable) { + console.warn( + `[realinfra] Postgres not reachable this run (${unreachableReason}) -- skipping honestly.`, + ); + ctx.skip(); + return; + } + const stamp = Date.now(); + const sourceId = `src_realpg_${stamp}`; + const tenantId = `t_realpg_${stamp}`; + + const writer = svc(); + await writer.registerSource({ + source_id: sourceId, + tenant_id: tenantId, + project_id: "p_1", + source_type: "api", + name: "Live Postgres pilot source", + access: "api", + created_by: "realinfra-test", + rights: { + license_id: "official-public-data", + allowed_purposes: ["company_enrichment"], + pii_allowed: false, + redistribution: "customer_deliverable", + verified_at: "2026-09-01", + verified_by: "realinfra-test", + }, + provenance: { + method: "official_api", + authority: "Bolagsverket", + cadence: "daily", + freshness_sla_hours: 24, + }, + }); + + // A BRAND NEW SourceRegistryService, own SourceService/repository + // construction under the hood, same DATABASE_URL -- pre-migration this + // would see NOTHING (each `new SourceService()` owned a private Map). + const reader = svc(); + const descriptor = await reader.describeSource(sourceId, tenantId); + expect(descriptor.source.source_id).toBe(sourceId); + expect(descriptor.source.tenant_id).toBe(tenantId); + expect(descriptor.rights?.license_id).toBe("official-public-data"); + expect(descriptor.rights?.allowed_purposes).toContain("company_enrichment"); + expect(descriptor.provenance?.method).toBe("official_api"); + expect(descriptor.provenance?.authority).toBe("Bolagsverket"); + + // Cross-checked with direct SQL -- real rows, not in-process bookkeeping. + const sourceRows = await adminSql!<{ source_id: string; tenant_id: string }[]>` + SELECT source_id, tenant_id FROM opt_sources WHERE source_id = ${sourceId}`; + expect(sourceRows).toHaveLength(1); + expect(sourceRows[0].tenant_id).toBe(tenantId); + + const rightsRows = await adminSql!<{ license_id: string; allowed_purposes: string[] }[]>` + SELECT license_id, allowed_purposes FROM opt_source_rights WHERE source_id = ${sourceId}`; + expect(rightsRows).toHaveLength(1); + expect(rightsRows[0].license_id).toBe("official-public-data"); + expect(rightsRows[0].allowed_purposes).toEqual(["company_enrichment"]); + }); + + it("registering the same source_id twice -> 'already exists', enforced by real Postgres (not app-side memory)", async (ctx) => { + if (!reachable) return ctx.skip(); + const stamp = Date.now(); + const sourceId = `src_realpg_dup_${stamp}`; + const tenantId = `t_realpg_dup_${stamp}`; + await svc().registerSource({ + source_id: sourceId, + tenant_id: tenantId, + project_id: "p_1", + source_type: "web", + name: "First", + }); + await expect( + svc().registerSource({ + source_id: sourceId, + tenant_id: tenantId, + project_id: "p_1", + source_type: "web", + name: "Second", + }), + ).rejects.toThrow(/already exists/i); + }); + + it("cross-tenant describe -> NOT_FOUND against real Postgres (§50, no existence oracle)", async (ctx) => { + if (!reachable) return ctx.skip(); + const stamp = Date.now(); + const sourceId = `src_realpg_tenant_${stamp}`; + await svc().registerSource({ + source_id: sourceId, + tenant_id: `t_a_realpg_${stamp}`, + project_id: "p_1", + source_type: "web", + name: "Tenant A source", + }); + await expect(svc().describeSource(sourceId, `t_b_realpg_${stamp}`)).rejects.toMatchObject({ + code: "NOT_FOUND", + }); + // And a foreign-tenant query never sees it either. + const foreign = await svc().querySources({ tenant_id: `t_b_realpg_${stamp}` }); + expect(foreign.find((s) => s.source_id === sourceId)).toBeUndefined(); + }); + + it("purpose gate fail-closed against real Postgres: no rights row denies every purpose", async (ctx) => { + if (!reachable) return ctx.skip(); + const stamp = Date.now(); + const sourceId = `src_realpg_norights_${stamp}`; + const tenantId = `t_realpg_${stamp}`; + await svc().registerSource({ + source_id: sourceId, + tenant_id: tenantId, + project_id: "p_1", + source_type: "web", + name: "No rights yet", + }); + await expect(svc().assertPurposeAllowed(sourceId, tenantId, "anything")).rejects.toMatchObject({ + code: "AUTHORIZATION_DENIED", + }); + }); + + it("health checks: bounded to the latest 100 rows in Postgres, consecutive-failure count survives across service instances", async (ctx) => { + if (!reachable) return ctx.skip(); + const stamp = Date.now(); + const sourceId = `src_realpg_health_${stamp}`; + const tenantId = `t_realpg_${stamp}`; + await svc().registerSource({ + source_id: sourceId, + tenant_id: tenantId, + project_id: "p_1", + source_type: "api", + name: "Health test source", + }); + + // 103 "down" checks (oldest -> newest), each via a FRESH service + // instance, simulating separate route-handler invocations. + for (let i = 0; i < 103; i++) { + await svc().recordHealthCheck({ + source_id: sourceId, + tenant_id: tenantId, + status: "down", + checked_at: new Date(Date.now() - (300 - i) * 1000).toISOString(), + }); + } + // Then 2 "up" checks, strictly more recent than every "down" above. + for (let i = 0; i < 2; i++) { + await svc().recordHealthCheck({ + source_id: sourceId, + tenant_id: tenantId, + status: "up", + checked_at: new Date(Date.now() - (2 - i) * 500).toISOString(), + latency_ms: 42, + }); + } + + // Bounded storage: never more than 100 rows for this source_id in Postgres. + const countRows = await adminSql!<{ n: number }[]>` + SELECT count(*)::int AS n FROM opt_source_health_checks WHERE source_id = ${sourceId}`; + expect(countRows[0].n).toBeLessThanOrEqual(100); + // All 103 "down" writes plus 2 "up" writes were made; some of the + // oldest "down" rows must have been pruned. + expect(countRows[0].n).toBeLessThan(105); + + const health = await svc().getHealth(sourceId, tenantId); + expect(health.status).toBe("up"); + expect(health.consecutive_failures).toBe(0); + + const freshness = await svc().getFreshness(sourceId, tenantId); + expect(freshness.fresh).toBe(true); + expect(freshness.last_success_at).toBeDefined(); + }); + + it("source.query filters by purpose and redistribution against real rows in opt_source_rights", async (ctx) => { + if (!reachable) return ctx.skip(); + const stamp = Date.now(); + const tenantId = `t_realpg_query_${stamp}`; + const sourceA = `src_realpg_a_${stamp}`; + const sourceB = `src_realpg_b_${stamp}`; + await svc().registerSource({ + source_id: sourceA, + tenant_id: tenantId, + project_id: "p_1", + source_type: "api", + name: "A", + rights: { + license_id: "l1", + allowed_purposes: ["company_enrichment"], + pii_allowed: false, + redistribution: "customer_deliverable", + verified_at: "2026-09-01", + verified_by: "t", + }, + }); + await svc().registerSource({ + source_id: sourceB, + tenant_id: tenantId, + project_id: "p_1", + source_type: "web", + name: "B", + rights: { + license_id: "l2", + allowed_purposes: ["compliance"], + pii_allowed: false, + redistribution: "internal", + verified_at: "2026-09-01", + verified_by: "t", + }, + }); + + const enrichment = await svc().querySources({ tenant_id: tenantId, purpose: "company_enrichment" }); + expect(enrichment.map((s) => s.source_id)).toEqual([sourceA]); + + const internalOnly = await svc().querySources({ tenant_id: tenantId, redistribution: "internal" }); + expect(internalOnly.map((s) => s.source_id)).toEqual([sourceB]); + }); + + it("setRights/setProvenance upsert in place (matching the Map.set replace semantics they replace)", async (ctx) => { + if (!reachable) return ctx.skip(); + const stamp = Date.now(); + const sourceId = `src_realpg_upsert_${stamp}`; + const tenantId = `t_realpg_${stamp}`; + const registry = svc(); + await registry.registerSource({ + source_id: sourceId, + tenant_id: tenantId, + project_id: "p_1", + source_type: "api", + name: "Upsert test", + }); + + await registry.setRights({ + source_id: sourceId, + tenant_id: tenantId, + license_id: "v1", + allowed_purposes: ["compliance"], + pii_allowed: false, + redistribution: "internal", + verified_at: "2026-09-01", + verified_by: "t", + }); + await registry.setRights({ + source_id: sourceId, + tenant_id: tenantId, + license_id: "v2", + allowed_purposes: ["compliance", "company_enrichment"], + pii_allowed: false, + redistribution: "customer_deliverable", + verified_at: "2026-09-02", + verified_by: "t2", + }); + + const rows = await adminSql!< + { license_id: string }[] + >`SELECT license_id FROM opt_source_rights WHERE source_id = ${sourceId}`; + expect(rows).toHaveLength(1); // one row, updated in place -- not two + expect(rows[0].license_id).toBe("v2"); + + const rights = await registry.getRights(sourceId, tenantId); + expect(rights?.license_id).toBe("v2"); + expect(rights?.allowed_purposes).toEqual(["compliance", "company_enrichment"]); + }); +}); diff --git a/tests/p0/source-registry.test.ts b/tests/p0/source-registry.test.ts index 1258ab5..2a1ff26 100644 --- a/tests/p0/source-registry.test.ts +++ b/tests/p0/source-registry.test.ts @@ -75,9 +75,11 @@ describe("SourceRegistryService — purpose gate (fail closed)", () => { it("denies a purpose outside the allowlist", async () => { const registry = svc(); await registerBase(registry); - await expect(registry.assertPurposeAllowed("src_bolagsverket", "t_a", "marketing")).rejects.toMatchObject({ - code: "AUTHORIZATION_DENIED", - }); + await expect(registry.assertPurposeAllowed("src_bolagsverket", "t_a", "marketing")).rejects.toMatchObject( + { + code: "AUTHORIZATION_DENIED", + }, + ); }); it("denies EVERY purpose when no rights record exists (fail closed)", async () => { @@ -129,7 +131,7 @@ describe("SourceRegistryService — health + freshness", () => { await registerBase(registry); const old = new Date(Date.now() - 72 * 3_600_000).toISOString(); // 72h > 48h SLA await registry.recordSuccess("src_bolagsverket", "t_a", old); - const freshness = registry.getFreshness("src_bolagsverket"); + const freshness = await registry.getFreshness("src_bolagsverket", "t_a"); expect(freshness.fresh).toBe(false); expect(freshness.age_hours).toBeGreaterThan(48); });