feat(data-os): Source Registry Postgres persistence + full HTTP route surface
Replaces the in-memory (per-instance, not even per-process-shared) Maps in SourceService/SourceRegistryService with real Postgres persistence, and adds the register/query/describe/health/rights/provenance/freshness HTTP routes that never existed (grep for SourceRegistryService in app/ previously returned nothing). - sql/002_source_registry.sql: opt_sources, opt_source_rights, opt_source_provenance, opt_source_health_checks, opt_source_freshness. Numbered 002 (not 006) because 004's opt_source_rate_policies already forward-references opt_sources, which no sql/ file defined until now — fixes 004's pre-existing broken migration ordering (opt_datasets remains a separate, unrelated, still-open gap in 004, flagged not fixed). - src/repositories/*: SourceRepository / SourceRegistryRepository seams, each with an in-memory implementation (business logic extracted verbatim) and a Postgres implementation, chosen via resolveStoreMode — production still fails hard with no DATABASE_URL (Master-DoD §3), but a configured production now actually works instead of always being blocked. - SourceService / SourceRegistryService: storage moved behind the repository seam; getHealth/getFreshness are now async and tenant-scoped at the accessor itself (needed once they're reachable directly over HTTP, not only after an earlier caller's tenant check). - src/services/source-registry.ts: shared singleton instances for the sources/* route group, fixing the "each route module owned its own private registry" defect (source-rate-policies and .../query previously couldn't see a source registered via POST /api/0pt/sources). - New routes: POST/GET /api/0pt/sources, GET /api/0pt/sources/[id], GET+PUT .../rights, GET+PUT .../provenance, GET+POST .../health, GET .../freshness. - Business logic unchanged: fail-closed purpose gate, cross-tenant NOT_FOUND (no existence oracle, §50), bounded 100-row health history. Tested against a real Postgres container (not mocked) — tests/p0/source-registry-postgres.realinfra.test.ts, env-gated on SOURCE_REGISTRY_TEST_DATABASE_URL, skips honestly when unset. All 249 pre-existing P0 tests + 7 new realinfra tests pass; also smoke-tested the full HTTP route surface end-to-end against next dev + live Postgres. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
f8bb0d5ab9
commit
07b13bd833
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* app/api/0pt/sources/[sourceId]/freshness/route.ts
|
||||
* =====================================================
|
||||
* Freshness view derived from provenance's freshness_sla_hours + the last
|
||||
* successful observation (recorded via POST .../health with status "up",
|
||||
* or SourceRegistryService.recordSuccess directly). Read-only — there is no
|
||||
* separate write path here by design; freshness is a derived signal, not
|
||||
* its own record.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { sourceRegistryService } from "@/src/services/source-registry";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const freshness = await sourceRegistryService.getFreshness(sourceId, tenantId);
|
||||
return NextResponse.json(freshness);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* app/api/0pt/sources/[sourceId]/health/route.ts
|
||||
* ==================================================
|
||||
* source.health — record a health observation (POST) and read the
|
||||
* aggregate view (GET): status, last check, consecutive failures. Bounded
|
||||
* history (latest 100 observations) — this is an operational signal, not
|
||||
* an audit log; AUD0 owns history.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, recordSourceHealthCheckSchema } from "@/src/validation/0pt-schemas";
|
||||
import { sourceRegistryService } from "@/src/services/source-registry";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const health = await sourceRegistryService.getHealth(sourceId, tenantId);
|
||||
return NextResponse.json(health);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const __raw = await request.json();
|
||||
const __parsed = parseBody(recordSourceHealthCheckSchema, __raw);
|
||||
if (!__parsed.ok) {
|
||||
return NextResponse.json(__parsed.body, { status: __parsed.status });
|
||||
}
|
||||
const body = __parsed.data;
|
||||
const health = await sourceRegistryService.recordHealthCheck({
|
||||
source_id: sourceId,
|
||||
tenant_id: body.tenant_id,
|
||||
status: body.status,
|
||||
checked_at: body.checked_at ?? new Date().toISOString(),
|
||||
latency_ms: body.latency_ms,
|
||||
detail: body.detail,
|
||||
});
|
||||
return NextResponse.json(health, { status: 201 });
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* app/api/0pt/sources/[sourceId]/provenance/route.ts
|
||||
* ======================================================
|
||||
* Per-SOURCE provenance/method metadata (Master-DoD §10) — HOW a source is
|
||||
* obtained (official_api | bulk_download | authorized_web | licensed_feed |
|
||||
* manual_upload), not per-field bridge_provenance. See BEFORE_STATE.md §3
|
||||
* for that distinction: this route does not, and cannot, close the
|
||||
* bridge_provenance gap — it only surfaces what SourceRegistryService
|
||||
* already models at the source level.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, setSourceProvenanceSchema } from "@/src/validation/0pt-schemas";
|
||||
import { sourceRegistryService } from "@/src/services/source-registry";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const provenance = await sourceRegistryService.getProvenance(sourceId, tenantId);
|
||||
if (!provenance) {
|
||||
return NextResponse.json({ error: `source ${sourceId} has no provenance record` }, { status: 404 });
|
||||
}
|
||||
return NextResponse.json(provenance);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function PUT(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const __raw = await request.json();
|
||||
const __parsed = parseBody(setSourceProvenanceSchema, __raw);
|
||||
if (!__parsed.ok) {
|
||||
return NextResponse.json(__parsed.body, { status: __parsed.status });
|
||||
}
|
||||
const body = __parsed.data;
|
||||
const provenance = await sourceRegistryService.setProvenance({
|
||||
...body,
|
||||
source_id: sourceId,
|
||||
updated_at: new Date().toISOString(),
|
||||
});
|
||||
return NextResponse.json(provenance);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -4,27 +4,24 @@
|
||||
* 0pt.query_source — bounded synchronous query execution for a registered
|
||||
* Source. Additive: does not modify sibling ../route.ts or ../../route.ts.
|
||||
*
|
||||
* NOTE — in-memory scope: this module instantiates its own SourceService,
|
||||
* matching the existing sources/route.ts and sources/[sourceId]/route.ts
|
||||
* pattern (each API route module owns a fresh in-memory registry, so state
|
||||
* is not shared across route modules yet — see the "Singleton note" in
|
||||
* src/services/document-capture-service.ts for prior art on this same repo
|
||||
* limitation). A source registered via POST /api/0pt/sources will not be
|
||||
* resolvable here until SourceService is backed by shared persistence.
|
||||
* Correction: this comment previously claimed `sql/002_p0_dataset_quality.sql`
|
||||
* already defined an `opt_sources` table for that. That file does not exist
|
||||
* and never has (git log --all: zero commits, any branch, ever). sql/ today
|
||||
* has only 001_init.sql and 004_iceberg_write_boundary.sql. The real fix —
|
||||
* backing SourceService with real persistence — is still open; no migration
|
||||
* for it exists yet.
|
||||
* Details: docs/P0_QUERY_SOURCE_FLOW.md.
|
||||
* UPDATED (Source Registry Postgres-persistence pass): now imports the
|
||||
* shared `sourceService` singleton from src/services/source-registry.ts
|
||||
* instead of constructing its own SourceService, closing the "not even a
|
||||
* shared process-wide singleton" gap BEFORE_STATE.md §1 flagged (a source
|
||||
* registered via POST /api/0pt/sources used to be invisible here). With
|
||||
* DATABASE_URL configured this is doubly fixed — state lives in Postgres
|
||||
* (sql/002_source_registry.sql), genuinely shared across processes, not
|
||||
* just within one. Details: docs/P0_QUERY_SOURCE_FLOW.md.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, sourceQuerySchema } from "@/src/validation/0pt-schemas";
|
||||
import { SourceService } from "@/src/services/source-service";
|
||||
import { SourceQueryService, SourceQueryTimeoutError, SourceQueryUnsupportedError } from "@/src/services/source-query-service";
|
||||
import { sourceService } from "@/src/services/source-registry";
|
||||
import {
|
||||
SourceQueryService,
|
||||
SourceQueryTimeoutError,
|
||||
SourceQueryUnsupportedError,
|
||||
} from "@/src/services/source-query-service";
|
||||
|
||||
const sourceService = new SourceService();
|
||||
const queryService = new SourceQueryService({ sourceService });
|
||||
|
||||
export async function POST(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* app/api/0pt/sources/[sourceId]/rights/route.ts
|
||||
* ==================================================
|
||||
* Rights / license / purpose metadata (Master-DoD §10). PUT is an upsert
|
||||
* (SourceRegistryService.setRights), matching the fail-closed purpose gate:
|
||||
* no rights record at all denies every purpose until one is set here.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, setSourceRightsSchema } from "@/src/validation/0pt-schemas";
|
||||
import { sourceRegistryService } from "@/src/services/source-registry";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const rights = await sourceRegistryService.getRights(sourceId, tenantId);
|
||||
if (!rights) {
|
||||
return NextResponse.json({ error: `source ${sourceId} has no rights record` }, { status: 404 });
|
||||
}
|
||||
return NextResponse.json(rights);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function PUT(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const __raw = await request.json();
|
||||
const __parsed = parseBody(setSourceRightsSchema, __raw);
|
||||
if (!__parsed.ok) {
|
||||
return NextResponse.json(__parsed.body, { status: __parsed.status });
|
||||
}
|
||||
const body = __parsed.data;
|
||||
const rights = await sourceRegistryService.setRights({ ...body, source_id: sourceId });
|
||||
return NextResponse.json(rights);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* app/api/0pt/sources/[sourceId]/route.ts
|
||||
* ==========================================
|
||||
* source.describe — full SourceDescriptor (base + rights + provenance +
|
||||
* health + freshness + rate policy). Cross-tenant probes get 404 NOT_FOUND
|
||||
* (no existence oracle, §50) via SourceRegistryService.describeSource's
|
||||
* assertSourceTenant gate.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { sourceRegistryService } from "@/src/services/source-registry";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const descriptor = await sourceRegistryService.describeSource(sourceId, tenantId);
|
||||
return NextResponse.json(descriptor);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,10 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { SourceRatePolicyService } from "@/src/services/source-rate-policy-service";
|
||||
import { sourceRatePolicyService as service } from "@/src/services/source-registry";
|
||||
|
||||
const service = new SourceRatePolicyService();
|
||||
|
||||
export async function GET(_request: Request, { params }: { params: Promise<{ sourceRatePolicyId: string }> }) {
|
||||
export async function GET(
|
||||
_request: Request,
|
||||
{ params }: { params: Promise<{ sourceRatePolicyId: string }> },
|
||||
) {
|
||||
try {
|
||||
const { sourceRatePolicyId } = await params;
|
||||
const policy = await service.getSourceRatePolicy(sourceRatePolicyId);
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, createSourceRatePolicySchema } from "@/src/validation/0pt-schemas";
|
||||
import { SourceRatePolicyService } from "@/src/services/source-rate-policy-service";
|
||||
|
||||
const service = new SourceRatePolicyService();
|
||||
import { sourceRatePolicyService as service } from "@/src/services/source-registry";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* app/api/0pt/sources/route.ts
|
||||
* ==============================
|
||||
* source.register (POST) + source.query (GET) — the P0 Source Registry
|
||||
* route surface BEFORE_STATE.md §1 found entirely missing
|
||||
* (`grep -rl "SourceRegistryService" app/` returned nothing). Backed by
|
||||
* SourceRegistryService, which now persists to Postgres when DATABASE_URL
|
||||
* is configured (src/repositories/*, sql/002_source_registry.sql).
|
||||
*
|
||||
* Uses the shared singleton from src/services/source-registry.ts, not a
|
||||
* fresh `new SourceRegistryService()` per route module — see that file's
|
||||
* doc comment.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, registerSourceSchema } from "@/src/validation/0pt-schemas";
|
||||
import { sourceRegistryService } from "@/src/services/source-registry";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
import type { SourceStatus, SourceType } from "@/src/types/source";
|
||||
import type { SourceRights } from "@/src/types/source-registry";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const __raw = await request.json();
|
||||
const __parsed = parseBody(registerSourceSchema, __raw);
|
||||
if (!__parsed.ok) {
|
||||
return NextResponse.json(__parsed.body, { status: __parsed.status });
|
||||
}
|
||||
const body = __parsed.data;
|
||||
const descriptor = await sourceRegistryService.registerSource(body);
|
||||
return NextResponse.json(descriptor, { status: 201 });
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const sources = await sourceRegistryService.querySources({
|
||||
tenant_id: tenantId,
|
||||
project_id: searchParams.get("project_id") ?? undefined,
|
||||
source_type: (searchParams.get("source_type") as SourceType) ?? undefined,
|
||||
status: (searchParams.get("status") as SourceStatus) ?? undefined,
|
||||
purpose: searchParams.get("purpose") ?? undefined,
|
||||
redistribution: (searchParams.get("redistribution") as SourceRights["redistribution"]) ?? undefined,
|
||||
});
|
||||
return NextResponse.json(sources);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
-- 0pt Source Registry persistence (Master-DoD §10; BEFORE_STATE.md §1).
|
||||
--
|
||||
-- Backs src/services/source-service.ts (0pt.source.schema.v1, the base
|
||||
-- registry) and src/services/source-registry-service.ts (rights/license/
|
||||
-- purpose, provenance, health, freshness — the P0 Source Registry surface
|
||||
-- layered on top). Both previously held their state in bare `new Map()`s,
|
||||
-- scoped per service *instance* (not even shared within one process) and
|
||||
-- hard-blocked from production boot by resolveStoreMode
|
||||
-- (src/platform/dependency-mode.ts) for lack of a DB. This migration is the
|
||||
-- DB half of closing that gap; src/repositories/* is the code half.
|
||||
--
|
||||
-- Deliberately numbered 002, not 006: sql/004_iceberg_write_boundary.sql's
|
||||
-- opt_source_rate_policies table already carries
|
||||
-- `source_id TEXT NOT NULL REFERENCES opt_sources (source_id)` — a
|
||||
-- forward reference to a table that has never existed in any sql/ file
|
||||
-- (opt_sources was, until this file, only ever a doc-comment aspiration —
|
||||
-- see the "What Postgres migration would require" section of
|
||||
-- BEFORE_STATE.md §1 and the retracted `sql/002_p0_dataset_quality.sql`
|
||||
-- claim in CLAUDE.md/app/api/0pt/sources/[sourceId]/query/route.ts, an
|
||||
-- UNRELATED fabricated claim about a different, never-written file that
|
||||
-- happened to reuse this same number). Applying 004 before opt_sources
|
||||
-- exists would fail outright on a fresh database; this file must run
|
||||
-- between 001 and 004 to make 004's existing FK resolvable. This is a
|
||||
-- pre-existing ordering bug in 004, not introduced here — fixed here only
|
||||
-- because this pass needed opt_sources to exist anyway.
|
||||
|
||||
-- ── 0pt.source.schema.v1 (src/types/source.ts) ─────────────────────────
|
||||
CREATE TABLE IF NOT EXISTS opt_sources (
|
||||
source_id TEXT PRIMARY KEY,
|
||||
tenant_id TEXT NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
source_type TEXT NOT NULL, -- database | file | web | api | stream | manual
|
||||
name TEXT NOT NULL,
|
||||
adapter_id TEXT, -- e.g. "fabric_onelake_sql" (src/adapters/fabric-onelake-adapter.ts)
|
||||
access TEXT, -- api | bulk | authorized_web | licensed (never a bypass mode)
|
||||
description TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'active', -- active | paused | archived | deprecated
|
||||
correlation_id TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ,
|
||||
created_by TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS opt_sources_tenant_project_idx ON opt_sources (tenant_id, project_id);
|
||||
CREATE INDEX IF NOT EXISTS opt_sources_type_idx ON opt_sources (source_type);
|
||||
CREATE INDEX IF NOT EXISTS opt_sources_status_idx ON opt_sources (status);
|
||||
|
||||
-- ── Rights / license / purpose metadata (src/types/source-registry.ts) ─
|
||||
-- One row per source (SourceRegistryService.setRights is an upsert-by-
|
||||
-- source_id, same as the Map it replaces). No rights row = fail-closed
|
||||
-- deny in assertPurposeAllowed — that logic lives in the service, not here.
|
||||
CREATE TABLE IF NOT EXISTS opt_source_rights (
|
||||
source_id TEXT PRIMARY KEY REFERENCES opt_sources (source_id),
|
||||
tenant_id TEXT NOT NULL,
|
||||
license_id TEXT NOT NULL,
|
||||
license_url TEXT,
|
||||
allowed_purposes TEXT[] NOT NULL DEFAULT '{}',
|
||||
pii_allowed BOOLEAN NOT NULL DEFAULT false,
|
||||
redistribution TEXT NOT NULL, -- internal | customer_deliverable | public | prohibited
|
||||
attribution TEXT,
|
||||
verified_at TEXT NOT NULL, -- ISO date the assessment was made (date-only in practice; kept as TEXT, not TIMESTAMPTZ, so it round-trips byte-for-byte)
|
||||
verified_by TEXT NOT NULL,
|
||||
notes TEXT,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS opt_source_rights_tenant_idx ON opt_source_rights (tenant_id);
|
||||
|
||||
-- ── Provenance / method metadata (per-SOURCE, not per-field — see
|
||||
-- BEFORE_STATE.md §3 for the distinction from bridge_provenance) ────────
|
||||
CREATE TABLE IF NOT EXISTS opt_source_provenance (
|
||||
source_id TEXT PRIMARY KEY REFERENCES opt_sources (source_id),
|
||||
tenant_id TEXT NOT NULL,
|
||||
method TEXT NOT NULL, -- official_api | bulk_download | authorized_web | licensed_feed | manual_upload
|
||||
authority TEXT,
|
||||
compliance_evidence_ref TEXT,
|
||||
cadence TEXT,
|
||||
freshness_sla_hours NUMERIC,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS opt_source_provenance_tenant_idx ON opt_source_provenance (tenant_id);
|
||||
|
||||
-- ── Health observations (source.health) — an operational signal, NOT an
|
||||
-- audit log (AUD0 owns history). Bounded to the latest 100 rows per
|
||||
-- source_id by the repository layer (src/repositories/postgres-source-
|
||||
-- registry-repository.ts), mirroring the in-memory implementation's
|
||||
-- `list.splice(0, list.length - 100)` bound.
|
||||
CREATE TABLE IF NOT EXISTS opt_source_health_checks (
|
||||
health_check_id BIGSERIAL PRIMARY KEY,
|
||||
source_id TEXT NOT NULL REFERENCES opt_sources (source_id),
|
||||
tenant_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL, -- up | degraded | down | unknown
|
||||
checked_at TIMESTAMPTZ NOT NULL,
|
||||
latency_ms INT,
|
||||
detail TEXT -- short operator-facing text; never a raw upstream body
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS opt_source_health_checks_source_idx ON opt_source_health_checks (source_id, checked_at DESC, health_check_id DESC);
|
||||
|
||||
-- ── Freshness: last successful observation per source (source.health's
|
||||
-- "up" checks call recordSuccess; getFreshness reads this + provenance's
|
||||
-- freshness_sla_hours). One row per source — upserted, not appended.
|
||||
CREATE TABLE IF NOT EXISTS opt_source_freshness (
|
||||
source_id TEXT PRIMARY KEY REFERENCES opt_sources (source_id),
|
||||
tenant_id TEXT NOT NULL,
|
||||
last_success_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS opt_source_freshness_tenant_idx ON opt_source_freshness (tenant_id);
|
||||
@@ -8,6 +8,18 @@
|
||||
-- different service's database. 0pt holds only the reference, never a copy
|
||||
-- of ST0RE's table/catalog/snapshot state (see the module doc comment on
|
||||
-- src/clients/st0re-lakehouse-client.ts).
|
||||
--
|
||||
-- Ordering note (added by the Source Registry Postgres-persistence pass):
|
||||
-- opt_source_rate_policies below has always forward-referenced
|
||||
-- `opt_sources (source_id)`, a table no sql/ file defined until
|
||||
-- sql/002_source_registry.sql landed -- applying this file to a fresh
|
||||
-- database before 002 would fail on that FK. 002 is now the one that must
|
||||
-- run first (001 -> 002 -> 004 -> 005). Flagging, not fixing here: this
|
||||
-- file's OTHER forward reference, `opt_dataset_iceberg_mappings.dataset_id
|
||||
-- REFERENCES opt_datasets (dataset_id)` below, has the identical problem --
|
||||
-- no sql/ file defines opt_datasets either -- and remains open; it is a
|
||||
-- dataset-registry gap, unrelated to and out of scope for the Source
|
||||
-- Registry work that added 002.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS opt_dataset_iceberg_mappings (
|
||||
dataset_iceberg_mapping_id TEXT PRIMARY KEY,
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* src/http/siax-error-response.ts
|
||||
* =================================
|
||||
* Maps a thrown error to a NextResponse for the source-registry routes
|
||||
* (app/api/0pt/sources/*). SourceRegistryService throws real `SiaxError`
|
||||
* instances (src/platform/errors.ts) with a pinned `.httpStatus`/`.code` —
|
||||
* this honors that mapping directly instead of re-deriving status codes from
|
||||
* regex-matching `error.message`, the ad-hoc pattern most other 0pt routes
|
||||
* use (app/api/0pt/documents/[documentId]/extraction/route.ts etc.) because
|
||||
* their services still throw plain `Error`. SourceService/SourceRegistryService's
|
||||
* lower-level plain Errors ("already exists", "not found") still fall back
|
||||
* to that same regex convention here, for consistency with the rest of the
|
||||
* repo's error surface.
|
||||
*/
|
||||
import { NextResponse } from "next/server";
|
||||
import { SiaxError } from "../platform/errors";
|
||||
|
||||
export function siaxErrorResponse(error: unknown): NextResponse {
|
||||
if (error instanceof SiaxError) {
|
||||
return NextResponse.json(error.toJSON(), { status: error.httpStatus });
|
||||
}
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
const status = /already exists/i.test(message) ? 409 : /not found/i.test(message) ? 404 : 400;
|
||||
return NextResponse.json({ error: message }, { status });
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* src/repositories/db.ts
|
||||
* ========================
|
||||
* Lazy Postgres connection for P0 capability repositories (src/repositories/*).
|
||||
*
|
||||
* Deliberately separate from lib/db.ts: that module is tuned for the
|
||||
* Vercel-hosted edge Postgres behind app/api/orders + app/api/subscriptions
|
||||
* (`ssl: "require"`, because that's what Vercel Postgres needs) and lives in
|
||||
* lib/, which src/ (the blocking-typechecked capability layer,
|
||||
* .dependency-cruiser.cjs's "no-src-importing-app-or-components" rule's
|
||||
* sibling concern) does not depend on. The P0 capability layer's Postgres
|
||||
* instance is the self-hosted VPS Postgres from docs/08-infrastructure.md
|
||||
* (infra/docker-compose.yml's `postgres` service, same private docker network
|
||||
* as the FastAPI engine, no TLS terminated in front of it) -- forcing
|
||||
* `ssl: "require"` here would break every real connection to it. SSL is
|
||||
* opt-in via PGSSL=require for deployments that do put TLS in front.
|
||||
*
|
||||
* Same lazy/env-guarded shape as lib/db.ts, and the same production-safety
|
||||
* decision as every other P0 dependency: src/platform/dependency-mode.ts's
|
||||
* `resolveStoreMode` is what actually decides postgres-vs-memory (and fails
|
||||
* hard in production with no URL) -- this module only opens the connection
|
||||
* once that decision has already been made.
|
||||
*/
|
||||
import postgres from "postgres";
|
||||
|
||||
export type RepositorySql = ReturnType<typeof postgres>;
|
||||
|
||||
/** DATABASE_URL, falling back to Vercel Postgres's env names -- same precedence lib/db.ts's dbUrl() uses, kept independent so src/ has no import on lib/. */
|
||||
export function repositoryDatabaseUrl(env: NodeJS.ProcessEnv = process.env): string | undefined {
|
||||
return env.DATABASE_URL || env.POSTGRES_URL || env.POSTGRES_PRISMA_URL;
|
||||
}
|
||||
|
||||
let _sql: RepositorySql | null = null;
|
||||
let _urlUsed: string | undefined;
|
||||
|
||||
/** Returns a memoized connection for `url`; re-opens if a different URL is requested (e.g. tests pointing at a fresh testcontainer). */
|
||||
export function getRepositorySql(url: string): RepositorySql {
|
||||
if (_sql && _urlUsed === url) return _sql;
|
||||
_sql = postgres(url, {
|
||||
prepare: false,
|
||||
idle_timeout: 20,
|
||||
max: 5,
|
||||
ssl: process.env.PGSSL === "require" ? "require" : false,
|
||||
});
|
||||
_urlUsed = url;
|
||||
return _sql;
|
||||
}
|
||||
|
||||
/** Test-only escape hatch: force the next getRepositorySql call to open a fresh connection. */
|
||||
export function resetRepositorySqlForTests(): void {
|
||||
_sql = null;
|
||||
_urlUsed = undefined;
|
||||
}
|
||||
|
||||
/** Test-only: gracefully close the memoized connection (so a vitest run doesn't hang on an open socket) and clear it. */
|
||||
export async function closeRepositorySqlForTests(): Promise<void> {
|
||||
const sql = _sql;
|
||||
_sql = null;
|
||||
_urlUsed = undefined;
|
||||
if (sql) await sql.end({ timeout: 5 });
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* src/repositories/in-memory-source-registry-repository.ts
|
||||
* ============================================================
|
||||
* The original SourceRegistryService storage (four Maps: rights,
|
||||
* provenance, healthChecks, lastSuccess), extracted verbatim behind the
|
||||
* SourceRegistryRepository seam. Health-check bounding (latest 100
|
||||
* observations) is preserved exactly; `listHealthChecks` returns
|
||||
* newest-first to match the Postgres implementation's `ORDER BY checked_at
|
||||
* DESC` contract (the original Map-based `getHealth` read `list[list.length
|
||||
* - 1]` for "most recent" and scanned backward for consecutive failures --
|
||||
* callers now get the same data pre-ordered instead).
|
||||
*/
|
||||
import type { SourceHealthCheck, SourceProvenance, SourceRights } from "../types/source-registry";
|
||||
import type { SourceRegistryRepository } from "./source-registry-repository";
|
||||
|
||||
export class InMemorySourceRegistryRepository implements SourceRegistryRepository {
|
||||
private readonly rights = new Map<string, SourceRights>();
|
||||
private readonly provenance = new Map<string, SourceProvenance>();
|
||||
private readonly healthChecks = new Map<string, SourceHealthCheck[]>();
|
||||
private readonly lastSuccess = new Map<string, string>();
|
||||
|
||||
async setRights(rights: SourceRights): Promise<void> {
|
||||
this.rights.set(rights.source_id, rights);
|
||||
}
|
||||
|
||||
async getRights(sourceId: string): Promise<SourceRights | undefined> {
|
||||
return this.rights.get(sourceId);
|
||||
}
|
||||
|
||||
async setProvenance(prov: SourceProvenance): Promise<void> {
|
||||
this.provenance.set(prov.source_id, prov);
|
||||
}
|
||||
|
||||
async getProvenance(sourceId: string): Promise<SourceProvenance | undefined> {
|
||||
return this.provenance.get(sourceId);
|
||||
}
|
||||
|
||||
async recordSuccess(sourceId: string, _tenantId: string, atIso: string): Promise<void> {
|
||||
this.lastSuccess.set(sourceId, atIso);
|
||||
}
|
||||
|
||||
async getLastSuccess(sourceId: string): Promise<string | undefined> {
|
||||
return this.lastSuccess.get(sourceId);
|
||||
}
|
||||
|
||||
async appendHealthCheck(check: SourceHealthCheck): Promise<void> {
|
||||
const list = this.healthChecks.get(check.source_id) ?? [];
|
||||
list.push(check);
|
||||
// Bounded history -- health is an operational signal, not an audit log
|
||||
// (AUD0 owns history); keep the latest 100 observations.
|
||||
if (list.length > 100) list.splice(0, list.length - 100);
|
||||
this.healthChecks.set(check.source_id, list);
|
||||
}
|
||||
|
||||
async listHealthChecks(sourceId: string, limit = 100): Promise<SourceHealthCheck[]> {
|
||||
const list = this.healthChecks.get(sourceId) ?? [];
|
||||
return [...list].reverse().slice(0, limit);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* src/repositories/in-memory-source-repository.ts
|
||||
* ==================================================
|
||||
* The original SourceService storage, extracted verbatim behind the
|
||||
* SourceRepository seam (src/repositories/source-repository.ts) so
|
||||
* PostgresSourceRepository can run the exact same business rules
|
||||
* ("already exists", "name is required", "not found", filter semantics)
|
||||
* against real Postgres instead. Used outside production when no
|
||||
* DATABASE_URL is configured (resolveStoreMode), and directly by tests that
|
||||
* want a hermetic, dependency-free SourceService.
|
||||
*/
|
||||
import type { CreateSourceInput, Source, SourceStatus, SourceType } from "../types/source";
|
||||
import type { SourceListFilter, SourceRepository } from "./source-repository";
|
||||
|
||||
/** Storage shape: nullable columns, exactly mirroring `opt_sources`. */
|
||||
interface SourceRecord {
|
||||
source_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
source_type: string;
|
||||
name: string;
|
||||
adapter_id: string | null;
|
||||
access: string | null;
|
||||
description: string | null;
|
||||
status: string;
|
||||
correlation_id: string | null;
|
||||
created_at: string;
|
||||
updated_at: string | null;
|
||||
created_by: string | null;
|
||||
}
|
||||
|
||||
export class InMemorySourceRepository implements SourceRepository {
|
||||
private sources: Map<string, SourceRecord> = new Map();
|
||||
|
||||
async create(input: CreateSourceInput): Promise<Source> {
|
||||
if (this.sources.has(input.source_id)) {
|
||||
throw new Error(`Source ${input.source_id} already exists`);
|
||||
}
|
||||
if (!input.name) {
|
||||
throw new Error("name is required");
|
||||
}
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const record: SourceRecord = {
|
||||
source_id: input.source_id,
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
source_type: input.source_type,
|
||||
name: input.name,
|
||||
adapter_id: input.adapter_id ?? null,
|
||||
access: input.access ?? null,
|
||||
description: input.description ?? null,
|
||||
status: input.status ?? "active",
|
||||
correlation_id: input.correlation_id ?? null,
|
||||
created_at: now,
|
||||
updated_at: null,
|
||||
created_by: input.created_by ?? null,
|
||||
};
|
||||
this.sources.set(record.source_id, record);
|
||||
return toSource(record);
|
||||
}
|
||||
|
||||
async get(sourceId: string): Promise<Source> {
|
||||
const record = this.sources.get(sourceId);
|
||||
if (!record) throw new Error(`Source ${sourceId} not found`);
|
||||
return toSource(record);
|
||||
}
|
||||
|
||||
async list(filter?: SourceListFilter): Promise<Source[]> {
|
||||
let results = Array.from(this.sources.values());
|
||||
if (filter?.tenant_id) results = results.filter((r) => r.tenant_id === filter.tenant_id);
|
||||
if (filter?.project_id) results = results.filter((r) => r.project_id === filter.project_id);
|
||||
if (filter?.source_type) results = results.filter((r) => r.source_type === filter.source_type);
|
||||
if (filter?.status) results = results.filter((r) => r.status === filter.status);
|
||||
return results.map(toSource);
|
||||
}
|
||||
}
|
||||
|
||||
function toSource(r: SourceRecord): Source {
|
||||
return {
|
||||
source_id: r.source_id,
|
||||
tenant_id: r.tenant_id,
|
||||
project_id: r.project_id,
|
||||
source_type: r.source_type as SourceType,
|
||||
name: r.name,
|
||||
adapter_id: r.adapter_id ?? undefined,
|
||||
access: (r.access as Source["access"]) ?? undefined,
|
||||
description: r.description ?? undefined,
|
||||
status: r.status as SourceStatus,
|
||||
correlation_id: r.correlation_id ?? undefined,
|
||||
created_at: r.created_at,
|
||||
updated_at: r.updated_at ?? undefined,
|
||||
created_by: r.created_by ?? undefined,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
/**
|
||||
* src/repositories/postgres-source-registry-repository.ts
|
||||
* ==========================================================
|
||||
* Real Postgres backing for SourceRegistryRepository, against
|
||||
* `opt_source_rights` / `opt_source_provenance` / `opt_source_health_checks`
|
||||
* / `opt_source_freshness` (sql/002_source_registry.sql). Rights and
|
||||
* provenance are one-row-per-source, upserted (matching the Map.set
|
||||
* replace-in-place semantics they replace); health checks are append-only,
|
||||
* bounded to the latest 100 rows per source_id after every insert; freshness
|
||||
* is a single upserted "last success" row per source.
|
||||
*/
|
||||
import type {
|
||||
SourceHealthCheck,
|
||||
SourceHealthStatus,
|
||||
SourceProvenance,
|
||||
SourceRights,
|
||||
} from "../types/source-registry";
|
||||
import type { RepositorySql } from "./db";
|
||||
import type { SourceRegistryRepository } from "./source-registry-repository";
|
||||
|
||||
function toIsoString(v: string | Date): string {
|
||||
return v instanceof Date ? v.toISOString() : v;
|
||||
}
|
||||
|
||||
interface RightsRow {
|
||||
source_id: string;
|
||||
tenant_id: string;
|
||||
license_id: string;
|
||||
license_url: string | null;
|
||||
allowed_purposes: string[];
|
||||
pii_allowed: boolean;
|
||||
redistribution: string;
|
||||
attribution: string | null;
|
||||
verified_at: string;
|
||||
verified_by: string;
|
||||
notes: string | null;
|
||||
}
|
||||
|
||||
function toRights(r: RightsRow): SourceRights {
|
||||
return {
|
||||
source_id: r.source_id,
|
||||
tenant_id: r.tenant_id,
|
||||
license_id: r.license_id,
|
||||
license_url: r.license_url ?? undefined,
|
||||
allowed_purposes: r.allowed_purposes ?? [],
|
||||
pii_allowed: r.pii_allowed,
|
||||
redistribution: r.redistribution as SourceRights["redistribution"],
|
||||
attribution: r.attribution ?? undefined,
|
||||
verified_at: r.verified_at,
|
||||
verified_by: r.verified_by,
|
||||
notes: r.notes ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
interface ProvenanceRow {
|
||||
source_id: string;
|
||||
tenant_id: string;
|
||||
method: string;
|
||||
authority: string | null;
|
||||
compliance_evidence_ref: string | null;
|
||||
cadence: string | null;
|
||||
freshness_sla_hours: string | number | null;
|
||||
updated_at: string | Date;
|
||||
}
|
||||
|
||||
function toProvenance(r: ProvenanceRow): SourceProvenance {
|
||||
return {
|
||||
source_id: r.source_id,
|
||||
tenant_id: r.tenant_id,
|
||||
method: r.method as SourceProvenance["method"],
|
||||
authority: r.authority ?? undefined,
|
||||
compliance_evidence_ref: r.compliance_evidence_ref ?? undefined,
|
||||
cadence: r.cadence ?? undefined,
|
||||
freshness_sla_hours: r.freshness_sla_hours === null ? undefined : Number(r.freshness_sla_hours),
|
||||
updated_at: toIsoString(r.updated_at),
|
||||
};
|
||||
}
|
||||
|
||||
interface HealthCheckRow {
|
||||
source_id: string;
|
||||
tenant_id: string;
|
||||
status: string;
|
||||
checked_at: string | Date;
|
||||
latency_ms: number | null;
|
||||
detail: string | null;
|
||||
}
|
||||
|
||||
function toHealthCheck(r: HealthCheckRow): SourceHealthCheck {
|
||||
return {
|
||||
source_id: r.source_id,
|
||||
tenant_id: r.tenant_id,
|
||||
status: r.status as SourceHealthStatus,
|
||||
checked_at: toIsoString(r.checked_at),
|
||||
latency_ms: r.latency_ms ?? undefined,
|
||||
detail: r.detail ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export class PostgresSourceRegistryRepository implements SourceRegistryRepository {
|
||||
constructor(private readonly sql: RepositorySql) {}
|
||||
|
||||
async setRights(rights: SourceRights): Promise<void> {
|
||||
await this.sql`
|
||||
INSERT INTO opt_source_rights
|
||||
(source_id, tenant_id, license_id, license_url, allowed_purposes, pii_allowed, redistribution, attribution, verified_at, verified_by, notes, updated_at)
|
||||
VALUES
|
||||
(${rights.source_id}, ${rights.tenant_id}, ${rights.license_id}, ${rights.license_url ?? null},
|
||||
${this.sql.array(rights.allowed_purposes)}, ${rights.pii_allowed}, ${rights.redistribution},
|
||||
${rights.attribution ?? null}, ${rights.verified_at}, ${rights.verified_by}, ${rights.notes ?? null}, now())
|
||||
ON CONFLICT (source_id) DO UPDATE SET
|
||||
tenant_id = EXCLUDED.tenant_id,
|
||||
license_id = EXCLUDED.license_id,
|
||||
license_url = EXCLUDED.license_url,
|
||||
allowed_purposes = EXCLUDED.allowed_purposes,
|
||||
pii_allowed = EXCLUDED.pii_allowed,
|
||||
redistribution = EXCLUDED.redistribution,
|
||||
attribution = EXCLUDED.attribution,
|
||||
verified_at = EXCLUDED.verified_at,
|
||||
verified_by = EXCLUDED.verified_by,
|
||||
notes = EXCLUDED.notes,
|
||||
updated_at = now()`;
|
||||
}
|
||||
|
||||
async getRights(sourceId: string): Promise<SourceRights | undefined> {
|
||||
const rows = await this.sql<RightsRow[]>`
|
||||
SELECT source_id, tenant_id, license_id, license_url, allowed_purposes, pii_allowed, redistribution, attribution, verified_at, verified_by, notes
|
||||
FROM opt_source_rights WHERE source_id = ${sourceId} LIMIT 1`;
|
||||
return rows[0] ? toRights(rows[0]) : undefined;
|
||||
}
|
||||
|
||||
async setProvenance(prov: SourceProvenance): Promise<void> {
|
||||
await this.sql`
|
||||
INSERT INTO opt_source_provenance
|
||||
(source_id, tenant_id, method, authority, compliance_evidence_ref, cadence, freshness_sla_hours, updated_at)
|
||||
VALUES
|
||||
(${prov.source_id}, ${prov.tenant_id}, ${prov.method}, ${prov.authority ?? null},
|
||||
${prov.compliance_evidence_ref ?? null}, ${prov.cadence ?? null}, ${prov.freshness_sla_hours ?? null}, ${prov.updated_at})
|
||||
ON CONFLICT (source_id) DO UPDATE SET
|
||||
tenant_id = EXCLUDED.tenant_id,
|
||||
method = EXCLUDED.method,
|
||||
authority = EXCLUDED.authority,
|
||||
compliance_evidence_ref = EXCLUDED.compliance_evidence_ref,
|
||||
cadence = EXCLUDED.cadence,
|
||||
freshness_sla_hours = EXCLUDED.freshness_sla_hours,
|
||||
updated_at = EXCLUDED.updated_at`;
|
||||
}
|
||||
|
||||
async getProvenance(sourceId: string): Promise<SourceProvenance | undefined> {
|
||||
const rows = await this.sql<ProvenanceRow[]>`
|
||||
SELECT source_id, tenant_id, method, authority, compliance_evidence_ref, cadence, freshness_sla_hours, updated_at
|
||||
FROM opt_source_provenance WHERE source_id = ${sourceId} LIMIT 1`;
|
||||
return rows[0] ? toProvenance(rows[0]) : undefined;
|
||||
}
|
||||
|
||||
async recordSuccess(sourceId: string, tenantId: string, atIso: string): Promise<void> {
|
||||
await this.sql`
|
||||
INSERT INTO opt_source_freshness (source_id, tenant_id, last_success_at)
|
||||
VALUES (${sourceId}, ${tenantId}, ${atIso})
|
||||
ON CONFLICT (source_id) DO UPDATE SET
|
||||
last_success_at = EXCLUDED.last_success_at,
|
||||
tenant_id = EXCLUDED.tenant_id`;
|
||||
}
|
||||
|
||||
async getLastSuccess(sourceId: string): Promise<string | undefined> {
|
||||
const rows = await this.sql<{ last_success_at: string | Date }[]>`
|
||||
SELECT last_success_at FROM opt_source_freshness WHERE source_id = ${sourceId} LIMIT 1`;
|
||||
return rows[0] ? toIsoString(rows[0].last_success_at) : undefined;
|
||||
}
|
||||
|
||||
async appendHealthCheck(check: SourceHealthCheck): Promise<void> {
|
||||
await this.sql`
|
||||
INSERT INTO opt_source_health_checks (source_id, tenant_id, status, checked_at, latency_ms, detail)
|
||||
VALUES (${check.source_id}, ${check.tenant_id}, ${check.status}, ${check.checked_at}, ${check.latency_ms ?? null}, ${check.detail ?? null})`;
|
||||
// Bounded history -- keep only the latest 100 observations per source
|
||||
// (health is an operational signal, not an audit log; AUD0 owns history),
|
||||
// same bound InMemorySourceRegistryRepository enforces on write.
|
||||
await this.sql`
|
||||
DELETE FROM opt_source_health_checks
|
||||
WHERE source_id = ${check.source_id}
|
||||
AND health_check_id NOT IN (
|
||||
SELECT health_check_id FROM opt_source_health_checks
|
||||
WHERE source_id = ${check.source_id}
|
||||
ORDER BY checked_at DESC, health_check_id DESC
|
||||
LIMIT 100
|
||||
)`;
|
||||
}
|
||||
|
||||
async listHealthChecks(sourceId: string, limit = 100): Promise<SourceHealthCheck[]> {
|
||||
const rows = await this.sql<HealthCheckRow[]>`
|
||||
SELECT source_id, tenant_id, status, checked_at, latency_ms, detail
|
||||
FROM opt_source_health_checks
|
||||
WHERE source_id = ${sourceId}
|
||||
ORDER BY checked_at DESC, health_check_id DESC
|
||||
LIMIT ${limit}`;
|
||||
return rows.map(toHealthCheck);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
/**
|
||||
* src/repositories/postgres-source-repository.ts
|
||||
* =================================================
|
||||
* Real Postgres backing for SourceRepository, against `opt_sources`
|
||||
* (sql/002_source_registry.sql). Same validation/error semantics as
|
||||
* InMemorySourceRepository ("already exists", "name is required", "not
|
||||
* found" -- exact message text preserved so every existing caller/test that
|
||||
* matches on those strings keeps working unmodified).
|
||||
*/
|
||||
import type { CreateSourceInput, Source, SourceStatus, SourceType } from "../types/source";
|
||||
import type { RepositorySql } from "./db";
|
||||
import type { SourceListFilter, SourceRepository } from "./source-repository";
|
||||
|
||||
interface SourceRow {
|
||||
source_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
source_type: string;
|
||||
name: string;
|
||||
adapter_id: string | null;
|
||||
access: string | null;
|
||||
description: string | null;
|
||||
status: string;
|
||||
correlation_id: string | null;
|
||||
created_at: string | Date;
|
||||
updated_at: string | Date | null;
|
||||
created_by: string | null;
|
||||
}
|
||||
|
||||
function toIsoString(v: string | Date): string {
|
||||
return v instanceof Date ? v.toISOString() : v;
|
||||
}
|
||||
|
||||
function toSource(r: SourceRow): Source {
|
||||
return {
|
||||
source_id: r.source_id,
|
||||
tenant_id: r.tenant_id,
|
||||
project_id: r.project_id,
|
||||
source_type: r.source_type as SourceType,
|
||||
name: r.name,
|
||||
adapter_id: r.adapter_id ?? undefined,
|
||||
access: (r.access as Source["access"]) ?? undefined,
|
||||
description: r.description ?? undefined,
|
||||
status: r.status as SourceStatus,
|
||||
correlation_id: r.correlation_id ?? undefined,
|
||||
created_at: toIsoString(r.created_at),
|
||||
updated_at: r.updated_at ? toIsoString(r.updated_at) : undefined,
|
||||
created_by: r.created_by ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export class PostgresSourceRepository implements SourceRepository {
|
||||
constructor(private readonly sql: RepositorySql) {}
|
||||
|
||||
async create(input: CreateSourceInput): Promise<Source> {
|
||||
if (!input.name) throw new Error("name is required");
|
||||
|
||||
const existing = await this.sql<{ source_id: string }[]>`
|
||||
SELECT source_id FROM opt_sources WHERE source_id = ${input.source_id} LIMIT 1`;
|
||||
if (existing.length > 0) {
|
||||
throw new Error(`Source ${input.source_id} already exists`);
|
||||
}
|
||||
|
||||
const rows = await this.sql<SourceRow[]>`
|
||||
INSERT INTO opt_sources
|
||||
(source_id, tenant_id, project_id, source_type, name, adapter_id, access, description, status, correlation_id, created_by)
|
||||
VALUES
|
||||
(${input.source_id}, ${input.tenant_id}, ${input.project_id}, ${input.source_type}, ${input.name},
|
||||
${input.adapter_id ?? null}, ${input.access ?? null}, ${input.description ?? null},
|
||||
${input.status ?? "active"}, ${input.correlation_id ?? null}, ${input.created_by ?? null})
|
||||
RETURNING source_id, tenant_id, project_id, source_type, name, adapter_id, access, description,
|
||||
status, correlation_id, created_at, updated_at, created_by`;
|
||||
return toSource(rows[0]);
|
||||
}
|
||||
|
||||
async get(sourceId: string): Promise<Source> {
|
||||
const rows = await this.sql<SourceRow[]>`
|
||||
SELECT source_id, tenant_id, project_id, source_type, name, adapter_id, access, description,
|
||||
status, correlation_id, created_at, updated_at, created_by
|
||||
FROM opt_sources WHERE source_id = ${sourceId} LIMIT 1`;
|
||||
if (!rows[0]) throw new Error(`Source ${sourceId} not found`);
|
||||
return toSource(rows[0]);
|
||||
}
|
||||
|
||||
async list(filter?: SourceListFilter): Promise<Source[]> {
|
||||
const rows = await this.sql<SourceRow[]>`
|
||||
SELECT source_id, tenant_id, project_id, source_type, name, adapter_id, access, description,
|
||||
status, correlation_id, created_at, updated_at, created_by
|
||||
FROM opt_sources
|
||||
WHERE 1=1
|
||||
${filter?.tenant_id ? this.sql`AND tenant_id = ${filter.tenant_id}` : this.sql``}
|
||||
${filter?.project_id ? this.sql`AND project_id = ${filter.project_id}` : this.sql``}
|
||||
${filter?.source_type ? this.sql`AND source_type = ${filter.source_type}` : this.sql``}
|
||||
${filter?.status ? this.sql`AND status = ${filter.status}` : this.sql``}
|
||||
ORDER BY created_at ASC`;
|
||||
return rows.map(toSource);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* src/repositories/source-registry-repository.ts
|
||||
* =================================================
|
||||
* Storage seam for the four Maps src/services/source-registry-service.ts
|
||||
* used to hold rights, provenance, health checks, and last-success
|
||||
* timestamps. Same split as source-repository.ts: one interface, an
|
||||
* in-memory implementation (extracted verbatim) and a Postgres
|
||||
* implementation (sql/002_source_registry.sql's `opt_source_rights`,
|
||||
* `opt_source_provenance`, `opt_source_health_checks`, `opt_source_freshness`
|
||||
* tables), chosen once via `resolveStoreMode`.
|
||||
*
|
||||
* Tenant isolation is NOT this layer's job -- exactly as before, when these
|
||||
* were bare Maps keyed only by source_id. SourceRegistryService's
|
||||
* `assertSourceTenant` gate is what enforces the cross-tenant NOT_FOUND
|
||||
* posture (§50), and it runs before every call into this repository. This
|
||||
* layer trusts the caller, same as the Maps did.
|
||||
*/
|
||||
import { resolveStoreMode } from "../platform/dependency-mode";
|
||||
import type { SourceHealthCheck, SourceProvenance, SourceRights } from "../types/source-registry";
|
||||
import { getRepositorySql, repositoryDatabaseUrl } from "./db";
|
||||
import { InMemorySourceRegistryRepository } from "./in-memory-source-registry-repository";
|
||||
import { PostgresSourceRegistryRepository } from "./postgres-source-registry-repository";
|
||||
|
||||
export interface SourceRegistryRepository {
|
||||
setRights(rights: SourceRights): Promise<void>;
|
||||
getRights(sourceId: string): Promise<SourceRights | undefined>;
|
||||
|
||||
setProvenance(prov: SourceProvenance): Promise<void>;
|
||||
getProvenance(sourceId: string): Promise<SourceProvenance | undefined>;
|
||||
|
||||
recordSuccess(sourceId: string, tenantId: string, atIso: string): Promise<void>;
|
||||
getLastSuccess(sourceId: string): Promise<string | undefined>;
|
||||
|
||||
appendHealthCheck(check: SourceHealthCheck): Promise<void>;
|
||||
/** Newest-first, bounded to the latest `limit` observations (default 100) -- health is an operational signal, not an audit log; AUD0 owns history. */
|
||||
listHealthChecks(sourceId: string, limit?: number): Promise<SourceHealthCheck[]>;
|
||||
}
|
||||
|
||||
export function createSourceRegistryRepository(databaseUrl?: string): SourceRegistryRepository {
|
||||
const mode = resolveStoreMode("0pt.source_registry.metadata", databaseUrl ?? repositoryDatabaseUrl());
|
||||
if (mode.mode === "postgres") {
|
||||
return new PostgresSourceRegistryRepository(getRepositorySql(mode.url));
|
||||
}
|
||||
return new InMemorySourceRegistryRepository();
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* src/repositories/source-repository.ts
|
||||
* =======================================
|
||||
* Storage seam for src/services/source-service.ts (0pt.source.schema.v1).
|
||||
* Extracted from what used to be a bare `new Map()` inside SourceService
|
||||
* itself so the same business logic (validation, "already exists"/"not
|
||||
* found" semantics, filtering) can run against either an in-memory Map
|
||||
* (tests, local dev, non-production with no DATABASE_URL) or real Postgres
|
||||
* (sql/002_source_registry.sql's `opt_sources` table) without the service
|
||||
* layer knowing which.
|
||||
*
|
||||
* `createSourceRepository` is the single choice point, delegating to
|
||||
* `resolveStoreMode` (src/platform/dependency-mode.ts) -- same
|
||||
* production-safety contract as every other P0 store: no DATABASE_URL in a
|
||||
* production runtime throws DEPENDENCY_UNAVAILABLE at construction rather
|
||||
* than silently serving the in-memory fallback.
|
||||
*/
|
||||
import { resolveStoreMode } from "../platform/dependency-mode";
|
||||
import type { CreateSourceInput, Source, SourceStatus, SourceType } from "../types/source";
|
||||
import { getRepositorySql, repositoryDatabaseUrl } from "./db";
|
||||
import { InMemorySourceRepository } from "./in-memory-source-repository";
|
||||
import { PostgresSourceRepository } from "./postgres-source-repository";
|
||||
|
||||
export interface SourceListFilter {
|
||||
tenant_id?: string;
|
||||
project_id?: string;
|
||||
source_type?: SourceType;
|
||||
status?: SourceStatus;
|
||||
}
|
||||
|
||||
export interface SourceRepository {
|
||||
create(input: CreateSourceInput): Promise<Source>;
|
||||
/** Throws (message contains "not found") when no such source exists -- matches the pre-migration Map behavior every caller already handles. */
|
||||
get(sourceId: string): Promise<Source>;
|
||||
list(filter?: SourceListFilter): Promise<Source[]>;
|
||||
}
|
||||
|
||||
export function createSourceRepository(databaseUrl?: string): SourceRepository {
|
||||
const mode = resolveStoreMode("0pt.source_registry.sources", databaseUrl ?? repositoryDatabaseUrl());
|
||||
if (mode.mode === "postgres") {
|
||||
return new PostgresSourceRepository(getRepositorySql(mode.url));
|
||||
}
|
||||
return new InMemorySourceRepository();
|
||||
}
|
||||
@@ -1,4 +1,8 @@
|
||||
import type { SourceRatePolicy, SourceRatePolicyStatus, CreateSourceRatePolicyInput } from "../types/source-rate-policy";
|
||||
import type {
|
||||
SourceRatePolicy,
|
||||
SourceRatePolicyStatus,
|
||||
CreateSourceRatePolicyInput,
|
||||
} from "../types/source-rate-policy";
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import { SourceService } from "./source-service";
|
||||
|
||||
@@ -33,7 +37,7 @@ export class SourceRatePolicyService {
|
||||
|
||||
constructor(opts?: { aud0BaseUrl?: string; sourceService?: SourceService }) {
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
this.sourceService = opts?.sourceService ?? new SourceService(opts?.aud0BaseUrl);
|
||||
this.sourceService = opts?.sourceService ?? new SourceService({ aud0BaseUrl: opts?.aud0BaseUrl });
|
||||
}
|
||||
|
||||
async createSourceRatePolicy(input: CreateSourceRatePolicyInput): Promise<SourceRatePolicy> {
|
||||
|
||||
@@ -14,20 +14,33 @@
|
||||
* Tenant isolation: every read path REQUIRES a tenant_id and filters on it —
|
||||
* a source registered by tenant A is invisible to tenant B (tested in
|
||||
* tests/p0/source-registry.test.ts, incl. the cross-tenant attack case §50).
|
||||
* `getHealth`/`getFreshness` now take tenant_id directly and run the same
|
||||
* `assertSourceTenant` gate every other accessor does — they are exposed as
|
||||
* standalone HTTP routes now (app/api/0pt/sources/[sourceId]/health,
|
||||
* .../freshness), so they can no longer rely on an earlier caller having
|
||||
* already checked tenancy on their behalf.
|
||||
*
|
||||
* Purpose enforcement: `assertPurposeAllowed` is the single hook other 0pt
|
||||
* capabilities call before using a source for a given purpose (SearchService
|
||||
* and DatasetService both do). No rights record ⇒ fail closed (deny).
|
||||
*
|
||||
* Persistence: in-memory Maps behind the same "pure core, thin registry"
|
||||
* shape as the sibling P0 services. Production boot is protected by
|
||||
* `resolveStoreMode` in the composition root (app/api routes construct with
|
||||
* the shared registry): with no database configured a production runtime
|
||||
* throws DEPENDENCY_UNAVAILABLE instead of serving Maps (Master-DoD §3).
|
||||
* Persistence: rights/provenance/health/freshness now live behind
|
||||
* SourceRegistryRepository (src/repositories/source-registry-repository.ts)
|
||||
* — real Postgres (`opt_source_rights`/`opt_source_provenance`/
|
||||
* `opt_source_health_checks`/`opt_source_freshness`,
|
||||
* sql/002_source_registry.sql) when DATABASE_URL is configured, in-memory
|
||||
* Maps otherwise. `resolveStoreMode` still fails production boot hard with
|
||||
* no DATABASE_URL (Master-DoD §3) — the difference from before is that a
|
||||
* configured production now actually WORKS (opt_sources/etc. exist), not
|
||||
* just fails safely.
|
||||
*/
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import { SiaxError } from "../platform/errors";
|
||||
import { SourceService } from "./source-service";
|
||||
import {
|
||||
createSourceRegistryRepository,
|
||||
type SourceRegistryRepository,
|
||||
} from "../repositories/source-registry-repository";
|
||||
import { SourceService, type SourceServiceOptions } from "./source-service";
|
||||
import { SourceRatePolicyService } from "./source-rate-policy-service";
|
||||
import type { CreateSourceInput, Source } from "../types/source";
|
||||
import type {
|
||||
@@ -46,21 +59,27 @@ export interface RegisterSourceInput extends CreateSourceInput {
|
||||
}
|
||||
|
||||
export class SourceRegistryService {
|
||||
private readonly rights = new Map<string, SourceRights>();
|
||||
private readonly provenance = new Map<string, SourceProvenance>();
|
||||
private readonly healthChecks = new Map<string, SourceHealthCheck[]>();
|
||||
private readonly lastSuccess = new Map<string, string>();
|
||||
private readonly repo: SourceRegistryRepository;
|
||||
private readonly aud0: Aud0Client;
|
||||
readonly sources: SourceService;
|
||||
readonly ratePolicies: SourceRatePolicyService;
|
||||
|
||||
constructor(opts?: {
|
||||
aud0BaseUrl?: string;
|
||||
/** Explicit DATABASE_URL override — mainly for tests. Falls back to resolveStoreMode's own env lookup when omitted. */
|
||||
databaseUrl?: string;
|
||||
sourceService?: SourceService;
|
||||
ratePolicyService?: SourceRatePolicyService;
|
||||
/** Inject a repository directly (tests) — bypasses resolveStoreMode entirely. */
|
||||
repository?: SourceRegistryRepository;
|
||||
}) {
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
this.sources = opts?.sourceService ?? new SourceService(opts?.aud0BaseUrl);
|
||||
this.repo = opts?.repository ?? createSourceRegistryRepository(opts?.databaseUrl);
|
||||
const sourceServiceOpts: SourceServiceOptions = {
|
||||
aud0BaseUrl: opts?.aud0BaseUrl,
|
||||
databaseUrl: opts?.databaseUrl,
|
||||
};
|
||||
this.sources = opts?.sourceService ?? new SourceService(sourceServiceOpts);
|
||||
this.ratePolicies =
|
||||
opts?.ratePolicyService ??
|
||||
new SourceRatePolicyService({ aud0BaseUrl: opts?.aud0BaseUrl, sourceService: this.sources });
|
||||
@@ -91,7 +110,7 @@ export class SourceRegistryService {
|
||||
// -------------------------------------------------------------------------
|
||||
async setRights(rights: SourceRights): Promise<SourceRights> {
|
||||
await this.assertSourceTenant(rights.source_id, rights.tenant_id);
|
||||
this.rights.set(rights.source_id, rights);
|
||||
await this.repo.setRights(rights);
|
||||
await this.emit("0pt.source_rights_set", rights.tenant_id, rights.source_id, {
|
||||
license_id: rights.license_id,
|
||||
allowed_purposes: rights.allowed_purposes,
|
||||
@@ -102,7 +121,7 @@ export class SourceRegistryService {
|
||||
|
||||
async getRights(sourceId: string, tenantId: string): Promise<SourceRights | undefined> {
|
||||
await this.assertSourceTenant(sourceId, tenantId);
|
||||
return this.rights.get(sourceId);
|
||||
return this.repo.getRights(sourceId);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -112,19 +131,19 @@ export class SourceRegistryService {
|
||||
*/
|
||||
async assertPurposeAllowed(sourceId: string, tenantId: string, purpose: string): Promise<SourceRights> {
|
||||
await this.assertSourceTenant(sourceId, tenantId);
|
||||
const rights = this.rights.get(sourceId);
|
||||
const rights = await this.repo.getRights(sourceId);
|
||||
if (!rights) {
|
||||
throw new SiaxError(
|
||||
"AUTHORIZATION_DENIED",
|
||||
"0pt.source_registry",
|
||||
`source ${sourceId} has no rights record — purpose "${purpose}" denied (fail closed)`
|
||||
`source ${sourceId} has no rights record — purpose "${purpose}" denied (fail closed)`,
|
||||
);
|
||||
}
|
||||
if (!rights.allowed_purposes.includes(purpose)) {
|
||||
throw new SiaxError(
|
||||
"AUTHORIZATION_DENIED",
|
||||
"0pt.source_registry",
|
||||
`purpose "${purpose}" is not in source ${sourceId}'s allowed purposes`
|
||||
`purpose "${purpose}" is not in source ${sourceId}'s allowed purposes`,
|
||||
);
|
||||
}
|
||||
return rights;
|
||||
@@ -135,7 +154,7 @@ export class SourceRegistryService {
|
||||
// -------------------------------------------------------------------------
|
||||
async setProvenance(prov: SourceProvenance): Promise<SourceProvenance> {
|
||||
await this.assertSourceTenant(prov.source_id, prov.tenant_id);
|
||||
this.provenance.set(prov.source_id, prov);
|
||||
await this.repo.setProvenance(prov);
|
||||
await this.emit("0pt.source_provenance_set", prov.tenant_id, prov.source_id, {
|
||||
method: prov.method,
|
||||
authority: prov.authority,
|
||||
@@ -146,18 +165,19 @@ export class SourceRegistryService {
|
||||
|
||||
async getProvenance(sourceId: string, tenantId: string): Promise<SourceProvenance | undefined> {
|
||||
await this.assertSourceTenant(sourceId, tenantId);
|
||||
return this.provenance.get(sourceId);
|
||||
return this.repo.getProvenance(sourceId);
|
||||
}
|
||||
|
||||
/** Record a successful data observation (drives freshness). */
|
||||
async recordSuccess(sourceId: string, tenantId: string, atIso?: string): Promise<void> {
|
||||
await this.assertSourceTenant(sourceId, tenantId);
|
||||
this.lastSuccess.set(sourceId, atIso ?? new Date().toISOString());
|
||||
await this.repo.recordSuccess(sourceId, tenantId, atIso ?? new Date().toISOString());
|
||||
}
|
||||
|
||||
getFreshness(sourceId: string, now: Date = new Date()): SourceFreshness {
|
||||
const prov = this.provenance.get(sourceId);
|
||||
const last = this.lastSuccess.get(sourceId);
|
||||
async getFreshness(sourceId: string, tenantId: string, now: Date = new Date()): Promise<SourceFreshness> {
|
||||
await this.assertSourceTenant(sourceId, tenantId);
|
||||
const prov = await this.repo.getProvenance(sourceId);
|
||||
const last = await this.repo.getLastSuccess(sourceId);
|
||||
const sla = prov?.freshness_sla_hours;
|
||||
if (!last) {
|
||||
return { source_id: sourceId, freshness_sla_hours: sla, fresh: sla === undefined };
|
||||
@@ -177,24 +197,21 @@ export class SourceRegistryService {
|
||||
// -------------------------------------------------------------------------
|
||||
async recordHealthCheck(check: SourceHealthCheck): Promise<SourceHealth> {
|
||||
await this.assertSourceTenant(check.source_id, check.tenant_id);
|
||||
const list = this.healthChecks.get(check.source_id) ?? [];
|
||||
list.push(check);
|
||||
// Bounded history — health is an operational signal, not an audit log
|
||||
// (AUD0 owns history); keep the latest 100 observations.
|
||||
if (list.length > 100) list.splice(0, list.length - 100);
|
||||
this.healthChecks.set(check.source_id, list);
|
||||
await this.repo.appendHealthCheck(check);
|
||||
if (check.status === "up") {
|
||||
await this.recordSuccess(check.source_id, check.tenant_id, check.checked_at);
|
||||
}
|
||||
return this.getHealth(check.source_id);
|
||||
return this.getHealth(check.source_id, check.tenant_id);
|
||||
}
|
||||
|
||||
getHealth(sourceId: string): SourceHealth {
|
||||
const list = this.healthChecks.get(sourceId) ?? [];
|
||||
const last = list[list.length - 1];
|
||||
async getHealth(sourceId: string, tenantId: string): Promise<SourceHealth> {
|
||||
await this.assertSourceTenant(sourceId, tenantId);
|
||||
// Newest-first (repository contract) — index 0 is the most recent check.
|
||||
const list = await this.repo.listHealthChecks(sourceId);
|
||||
const last = list[0];
|
||||
let consecutiveFailures = 0;
|
||||
for (let i = list.length - 1; i >= 0; i--) {
|
||||
if (list[i].status === "down" || list[i].status === "degraded") consecutiveFailures++;
|
||||
for (const check of list) {
|
||||
if (check.status === "down" || check.status === "degraded") consecutiveFailures++;
|
||||
else break;
|
||||
}
|
||||
const status: SourceHealthStatus = last?.status ?? "unknown";
|
||||
@@ -223,27 +240,41 @@ export class SourceRegistryService {
|
||||
source_type: filter.source_type,
|
||||
status: filter.status,
|
||||
});
|
||||
if (filter.purpose !== undefined) {
|
||||
results = results.filter((s) => {
|
||||
const rights = this.rights.get(s.source_id);
|
||||
return rights !== undefined && rights.allowed_purposes.includes(filter.purpose as string);
|
||||
});
|
||||
}
|
||||
if (filter.redistribution !== undefined) {
|
||||
results = results.filter((s) => this.rights.get(s.source_id)?.redistribution === filter.redistribution);
|
||||
if (filter.purpose !== undefined || filter.redistribution !== undefined) {
|
||||
const rightsEntries = await Promise.all(
|
||||
results.map(async (s) => [s.source_id, await this.repo.getRights(s.source_id)] as const),
|
||||
);
|
||||
const rightsBySource = new Map(rightsEntries);
|
||||
if (filter.purpose !== undefined) {
|
||||
results = results.filter((s) => {
|
||||
const rights = rightsBySource.get(s.source_id);
|
||||
return rights !== undefined && rights.allowed_purposes.includes(filter.purpose as string);
|
||||
});
|
||||
}
|
||||
if (filter.redistribution !== undefined) {
|
||||
results = results.filter(
|
||||
(s) => rightsBySource.get(s.source_id)?.redistribution === filter.redistribution,
|
||||
);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async describeSource(sourceId: string, tenantId: string): Promise<SourceDescriptor> {
|
||||
const source = await this.assertSourceTenant(sourceId, tenantId);
|
||||
const policies = await this.ratePolicies.listSourceRatePolicies({ tenant_id: tenantId, source_id: sourceId });
|
||||
const [policies, rights, provenance, health, freshness] = await Promise.all([
|
||||
this.ratePolicies.listSourceRatePolicies({ tenant_id: tenantId, source_id: sourceId }),
|
||||
this.repo.getRights(sourceId),
|
||||
this.repo.getProvenance(sourceId),
|
||||
this.getHealth(sourceId, tenantId),
|
||||
this.getFreshness(sourceId, tenantId),
|
||||
]);
|
||||
return {
|
||||
source,
|
||||
rights: this.rights.get(sourceId),
|
||||
provenance: this.provenance.get(sourceId),
|
||||
health: this.getHealth(sourceId),
|
||||
freshness: this.getFreshness(sourceId),
|
||||
rights,
|
||||
provenance,
|
||||
health,
|
||||
freshness,
|
||||
rate_policy: policies[0],
|
||||
};
|
||||
}
|
||||
@@ -276,7 +307,7 @@ export class SourceRegistryService {
|
||||
eventType: string,
|
||||
tenantId: string,
|
||||
sourceId: string,
|
||||
payload: Record<string, unknown>
|
||||
payload: Record<string, unknown>,
|
||||
): Promise<void> {
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_${eventType}_${sourceId}_${Date.now()}`,
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
/**
|
||||
* src/services/source-registry.ts
|
||||
* =================================
|
||||
* Shared singleton instances for the source-registry route group
|
||||
* (app/api/0pt/sources/*), matching src/services/lakehouse-mapping-registry.ts's
|
||||
* precedent.
|
||||
*
|
||||
* BEFORE_STATE.md §1 flagged this as materially worse than plain
|
||||
* in-memory-per-module: each route file previously constructed its own
|
||||
* `new SourceService()` (and `new SourceRatePolicyService()`), so a source
|
||||
* registered via one route module was invisible to another within the same
|
||||
* running process — "not even a shared process-wide singleton". Once
|
||||
* DATABASE_URL is configured this stops mattering (state lives in Postgres,
|
||||
* genuinely shared across processes/deploys — see src/repositories/*), but
|
||||
* it still matters for the in-memory (no DATABASE_URL) dev/test mode, where
|
||||
* state must be shared within one process to be usable at all. Constructing
|
||||
* all three here once, wired together, and importing them everywhere fixes
|
||||
* both call sites this repo already had (sources/rate-policies/*,
|
||||
* sources/[sourceId]/query) plus every new route this pass adds.
|
||||
*/
|
||||
import { SourceService } from "./source-service";
|
||||
import { SourceRatePolicyService } from "./source-rate-policy-service";
|
||||
import { SourceRegistryService } from "./source-registry-service";
|
||||
|
||||
export const sourceService = new SourceService();
|
||||
export const sourceRatePolicyService = new SourceRatePolicyService({ sourceService });
|
||||
export const sourceRegistryService = new SourceRegistryService({
|
||||
sourceService,
|
||||
ratePolicyService: sourceRatePolicyService,
|
||||
});
|
||||
@@ -7,107 +7,74 @@
|
||||
* and `SourceRatePolicyService` (src/services/source-rate-policy-service.ts)
|
||||
* both look a `source_id` up against — see docs/P0_QUERY_SOURCE_FLOW.md.
|
||||
*
|
||||
* Deliberately small and in-memory, same "pure core, thin registry" shape
|
||||
* every other P0 service in this repo already uses; a real deployment backs
|
||||
* this with `opt_sources`. In-memory scope: each `new SourceService()` owns
|
||||
* its own registry — the pre-existing limitation docs/P0_QUERY_SOURCE_FLOW.md's
|
||||
* "Known gaps" section documents (a Source registered via one API route module
|
||||
* is not yet visible to another route module in the same running process).
|
||||
* Callers that need a shared view pass an existing instance in via the
|
||||
* `sourceService` constructor option, exactly like SourceQueryService and
|
||||
* SourceRatePolicyService already do.
|
||||
* Persistence: storage now lives behind SourceRepository
|
||||
* (src/repositories/source-repository.ts) — real Postgres (`opt_sources`,
|
||||
* sql/002_source_registry.sql) when DATABASE_URL is configured, an
|
||||
* in-memory Map otherwise (non-production only; resolveStoreMode fails hard
|
||||
* in production with no URL). This service still owns the business rules
|
||||
* (validation, tenant-agnostic CRUD orchestration) and the AUD0 emission —
|
||||
* only the storage moved.
|
||||
*
|
||||
* Registering a source emits a best-effort AUD0 `0pt.source_created` event —
|
||||
* the same "no mutation without an evented trail" doctrine every other P0
|
||||
* registry service follows. With no AUD0_URL configured that emission is a
|
||||
* no-op (see aud0-client.ts), so this stays hermetic in tests and local dev.
|
||||
* Callers that need a shared view across route modules should use the
|
||||
* shared singleton in src/services/source-registry.ts rather than
|
||||
* constructing a fresh `new SourceService()` per module — see that file's
|
||||
* doc comment for the "each route owned its own registry" history this
|
||||
* fixes.
|
||||
*/
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import {
|
||||
createSourceRepository,
|
||||
type SourceRepository,
|
||||
type SourceListFilter,
|
||||
} from "../repositories/source-repository";
|
||||
import type { CreateSourceInput, Source, SourceStatus, SourceType } from "../types/source";
|
||||
|
||||
/**
|
||||
* Storage shape: nullable columns, exactly as the eventual `opt_sources` table
|
||||
* would hold them. `toSource()` converts back to the `Source` DTO (undefined
|
||||
* rather than null) so callers never have to handle both.
|
||||
*/
|
||||
interface SourceRecord {
|
||||
source_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
source_type: string;
|
||||
name: string;
|
||||
adapter_id: string | null;
|
||||
access: string | null;
|
||||
description: string | null;
|
||||
status: string;
|
||||
correlation_id: string | null;
|
||||
created_at: string;
|
||||
updated_at: string | null;
|
||||
created_by: string | null;
|
||||
export interface SourceServiceOptions {
|
||||
aud0BaseUrl?: string;
|
||||
/** Explicit DATABASE_URL override — mainly for tests pointing at a specific Postgres instance. Falls back to resolveStoreMode's own env lookup when omitted. */
|
||||
databaseUrl?: string;
|
||||
/** Inject a repository directly (tests) — bypasses resolveStoreMode entirely. */
|
||||
repository?: SourceRepository;
|
||||
}
|
||||
|
||||
export class SourceService {
|
||||
private sources: Map<string, SourceRecord> = new Map();
|
||||
private aud0: Aud0Client;
|
||||
private readonly repo: SourceRepository;
|
||||
private readonly aud0: Aud0Client;
|
||||
|
||||
constructor(aud0BaseUrl?: string) {
|
||||
this.aud0 = createAud0Client(aud0BaseUrl);
|
||||
constructor(opts?: SourceServiceOptions) {
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
this.repo = opts?.repository ?? createSourceRepository(opts?.databaseUrl);
|
||||
}
|
||||
|
||||
async createSource(input: CreateSourceInput): Promise<Source> {
|
||||
if (this.sources.has(input.source_id)) {
|
||||
throw new Error(`Source ${input.source_id} already exists`);
|
||||
}
|
||||
if (!input.name) {
|
||||
throw new Error("name is required");
|
||||
}
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const record: SourceRecord = {
|
||||
source_id: input.source_id,
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
source_type: input.source_type,
|
||||
name: input.name,
|
||||
adapter_id: input.adapter_id ?? null,
|
||||
access: input.access ?? null,
|
||||
description: input.description ?? null,
|
||||
status: input.status ?? "active",
|
||||
correlation_id: input.correlation_id ?? null,
|
||||
created_at: now,
|
||||
updated_at: null,
|
||||
created_by: input.created_by ?? null,
|
||||
};
|
||||
this.sources.set(record.source_id, record);
|
||||
const source = await this.repo.create(input);
|
||||
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_source_created_${record.source_id}`,
|
||||
event_id: `evt_0pt_source_created_${source.source_id}`,
|
||||
event_type: "0pt.source_created",
|
||||
tenant_id: record.tenant_id,
|
||||
project_id: record.project_id,
|
||||
actor_id: record.created_by ?? "system",
|
||||
tenant_id: source.tenant_id,
|
||||
project_id: source.project_id,
|
||||
actor_id: source.created_by ?? "system",
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.create_source",
|
||||
resource_type: "source",
|
||||
resource_id: record.source_id,
|
||||
resource_id: source.source_id,
|
||||
risk_level: "L2",
|
||||
payload: {
|
||||
source_type: record.source_type,
|
||||
name: record.name,
|
||||
adapter_id: record.adapter_id,
|
||||
access: record.access,
|
||||
source_type: source.source_type,
|
||||
name: source.name,
|
||||
adapter_id: source.adapter_id,
|
||||
access: source.access,
|
||||
},
|
||||
created_at: now,
|
||||
correlation_id: record.correlation_id ?? undefined,
|
||||
created_at: source.created_at,
|
||||
correlation_id: source.correlation_id,
|
||||
});
|
||||
|
||||
return this.toSource(record);
|
||||
return source;
|
||||
}
|
||||
|
||||
async getSource(sourceId: string): Promise<Source> {
|
||||
const record = this.sources.get(sourceId);
|
||||
if (!record) throw new Error(`Source ${sourceId} not found`);
|
||||
return this.toSource(record);
|
||||
return this.repo.get(sourceId);
|
||||
}
|
||||
|
||||
async listSources(filter?: {
|
||||
@@ -116,29 +83,6 @@ export class SourceService {
|
||||
source_type?: SourceType;
|
||||
status?: SourceStatus;
|
||||
}): Promise<Source[]> {
|
||||
let results = Array.from(this.sources.values());
|
||||
if (filter?.tenant_id) results = results.filter((r) => r.tenant_id === filter.tenant_id);
|
||||
if (filter?.project_id) results = results.filter((r) => r.project_id === filter.project_id);
|
||||
if (filter?.source_type) results = results.filter((r) => r.source_type === filter.source_type);
|
||||
if (filter?.status) results = results.filter((r) => r.status === filter.status);
|
||||
return results.map((r) => this.toSource(r));
|
||||
}
|
||||
|
||||
private toSource(r: SourceRecord): Source {
|
||||
return {
|
||||
source_id: r.source_id,
|
||||
tenant_id: r.tenant_id,
|
||||
project_id: r.project_id,
|
||||
source_type: r.source_type as SourceType,
|
||||
name: r.name,
|
||||
adapter_id: r.adapter_id ?? undefined,
|
||||
access: (r.access as Source["access"]) ?? undefined,
|
||||
description: r.description ?? undefined,
|
||||
status: r.status as SourceStatus,
|
||||
correlation_id: r.correlation_id ?? undefined,
|
||||
created_at: r.created_at,
|
||||
updated_at: r.updated_at ?? undefined,
|
||||
created_by: r.created_by ?? undefined,
|
||||
};
|
||||
return this.repo.list(filter as SourceListFilter | undefined);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,6 +95,64 @@ export const createSourceRatePolicySchema = z.object({
|
||||
created_by: z.string().min(1).optional(),
|
||||
});
|
||||
|
||||
/** Nested `rights` block on POST /api/0pt/sources, and the body of PUT /api/0pt/sources/[sourceId]/rights -- mirrors SourceRights (minus source_id/tenant_id, which come from the URL/outer body). */
|
||||
const sourceRightsInputSchema = z.object({
|
||||
license_id: z.string().min(1),
|
||||
license_url: z.string().min(1).optional(),
|
||||
allowed_purposes: z.array(z.string().min(1)).default([]),
|
||||
pii_allowed: z.boolean(),
|
||||
redistribution: z.enum(['internal', 'customer_deliverable', 'public', 'prohibited']),
|
||||
attribution: z.string().min(1).optional(),
|
||||
verified_at: z.string().min(1),
|
||||
verified_by: z.string().min(1),
|
||||
notes: z.string().min(1).optional(),
|
||||
});
|
||||
|
||||
/** Nested `provenance` block on POST /api/0pt/sources, and the body of PUT /api/0pt/sources/[sourceId]/provenance -- mirrors SourceProvenance (minus source_id/tenant_id/updated_at, which are set by the service). */
|
||||
const sourceProvenanceInputSchema = z.object({
|
||||
method: z.enum(['official_api', 'bulk_download', 'authorized_web', 'licensed_feed', 'manual_upload']),
|
||||
authority: z.string().min(1).optional(),
|
||||
compliance_evidence_ref: z.string().min(1).optional(),
|
||||
cadence: z.string().min(1).optional(),
|
||||
freshness_sla_hours: z.number().nonnegative().optional(),
|
||||
});
|
||||
|
||||
/** POST /api/0pt/sources -- source.register. */
|
||||
export const registerSourceSchema = z.object({
|
||||
source_id: z.string().min(1),
|
||||
tenant_id: z.string().min(1),
|
||||
project_id: z.string().min(1),
|
||||
source_type: z.enum(['database', 'file', 'web', 'api', 'stream', 'manual']),
|
||||
name: z.string().min(1),
|
||||
adapter_id: z.string().min(1).optional(),
|
||||
access: z.enum(['api', 'bulk', 'authorized_web', 'licensed']).optional(),
|
||||
description: z.string().min(1).optional(),
|
||||
status: z.enum(['active', 'paused', 'archived', 'deprecated']).optional(),
|
||||
correlation_id: z.string().min(1).optional(),
|
||||
created_by: z.string().min(1).optional(),
|
||||
rights: sourceRightsInputSchema.optional(),
|
||||
provenance: sourceProvenanceInputSchema.optional(),
|
||||
});
|
||||
|
||||
/** PUT /api/0pt/sources/[sourceId]/rights -- source.set_rights. */
|
||||
export const setSourceRightsSchema = sourceRightsInputSchema.extend({
|
||||
tenant_id: z.string().min(1),
|
||||
});
|
||||
|
||||
/** PUT /api/0pt/sources/[sourceId]/provenance -- source.set_provenance. */
|
||||
export const setSourceProvenanceSchema = sourceProvenanceInputSchema.extend({
|
||||
tenant_id: z.string().min(1),
|
||||
});
|
||||
|
||||
/** POST /api/0pt/sources/[sourceId]/health -- source.health (record a health check). */
|
||||
export const recordSourceHealthCheckSchema = z.object({
|
||||
tenant_id: z.string().min(1),
|
||||
status: z.enum(['up', 'degraded', 'down', 'unknown']),
|
||||
checked_at: z.string().min(1).optional(),
|
||||
latency_ms: z.number().nonnegative().optional(),
|
||||
detail: z.string().min(1).optional(),
|
||||
});
|
||||
|
||||
/** POST /api/0pt/lakehouse/mappings -- mirrors MapDatasetToIcebergTableInput. */
|
||||
export const mapDatasetToIcebergTableSchema = z.object({
|
||||
tenant_id: z.string().min(1),
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
/**
|
||||
* REAL Postgres integration test (Master-DoD §48/§50) for the Source
|
||||
* Registry's new persistence layer (src/repositories/*,
|
||||
* sql/002_source_registry.sql) -- the P010 task's central deliverable: this
|
||||
* is what proves the Map -> Postgres migration actually works, not just
|
||||
* that it typechecks.
|
||||
*
|
||||
* Same honest-degrade convention as tests/p0/iceberg-write-service.realinfra.test.ts
|
||||
* and tests/integration/analytics-clickhouse.integration.test.ts: gated on
|
||||
* an env var (SOURCE_REGISTRY_TEST_DATABASE_URL), SKIPS HONESTLY when unset,
|
||||
* never fabricates success. Point it at a disposable Postgres, e.g.:
|
||||
*
|
||||
* docker run -d --name opt-source-registry-pgtest \
|
||||
* -e POSTGRES_USER=opt -e POSTGRES_PASSWORD=opt_test_pw -e POSTGRES_DB=opt_test \
|
||||
* -p 15433:5432 postgres:16-alpine
|
||||
* SOURCE_REGISTRY_TEST_DATABASE_URL=postgres://opt:opt_test_pw@localhost:15433/opt_test \
|
||||
* pnpm vitest run tests/p0/source-registry-postgres.realinfra.test.ts
|
||||
*
|
||||
* This is NOT a mock-repository test -- every assertion below is driven
|
||||
* through SourceRegistryService with `databaseUrl` set, i.e. the real
|
||||
* PostgresSourceRepository/PostgresSourceRegistryRepository HTTP-free SQL
|
||||
* path (postgres.js against the live container), with a handful of
|
||||
* assertions cross-checked with direct SQL against the same database to
|
||||
* prove the rows are really there, not an artifact of the service's own
|
||||
* in-process bookkeeping.
|
||||
*
|
||||
* beforeAll applies sql/002_source_registry.sql itself (every statement is
|
||||
* `CREATE TABLE/INDEX IF NOT EXISTS`, so this is idempotent and safe to run
|
||||
* against an already-migrated database too) -- this test does not require
|
||||
* the operator to have pre-run migrations.
|
||||
*/
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import postgres from "postgres";
|
||||
import { SourceRegistryService } from "@/src/services/source-registry-service";
|
||||
import { closeRepositorySqlForTests } from "@/src/repositories/db";
|
||||
|
||||
const TEST_DATABASE_URL = process.env.SOURCE_REGISTRY_TEST_DATABASE_URL;
|
||||
const run = TEST_DATABASE_URL ? describe : describe.skip;
|
||||
|
||||
let adminSql: ReturnType<typeof postgres> | undefined;
|
||||
let reachable = false;
|
||||
let unreachableReason = "";
|
||||
|
||||
beforeAll(async () => {
|
||||
if (!TEST_DATABASE_URL) {
|
||||
console.warn(
|
||||
"[realinfra] SOURCE_REGISTRY_TEST_DATABASE_URL not set this run -- skipping honestly, not fabricating success.",
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
adminSql = postgres(TEST_DATABASE_URL, { prepare: false, ssl: false, max: 2 });
|
||||
await Promise.race([
|
||||
adminSql`SELECT 1`,
|
||||
new Promise((_, reject) => setTimeout(() => reject(new Error("connect timeout")), 5000)),
|
||||
]);
|
||||
const migrationPath = path.join(process.cwd(), "sql", "002_source_registry.sql");
|
||||
const migration = fs.readFileSync(migrationPath, "utf8");
|
||||
await adminSql.unsafe(migration);
|
||||
reachable = true;
|
||||
} catch (err) {
|
||||
reachable = false;
|
||||
unreachableReason = err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
afterAll(async () => {
|
||||
await adminSql?.end({ timeout: 5 });
|
||||
await closeRepositorySqlForTests();
|
||||
});
|
||||
|
||||
function svc(): SourceRegistryService {
|
||||
return new SourceRegistryService({ databaseUrl: TEST_DATABASE_URL });
|
||||
}
|
||||
|
||||
run("SourceRegistryService against a LIVE Postgres (sql/002_source_registry.sql)", () => {
|
||||
it("persists across independent service instances -- the exact bug this migration fixes (in-memory Maps were per-instance, not even per-process)", async (ctx) => {
|
||||
if (!reachable) {
|
||||
console.warn(
|
||||
`[realinfra] Postgres not reachable this run (${unreachableReason}) -- skipping honestly.`,
|
||||
);
|
||||
ctx.skip();
|
||||
return;
|
||||
}
|
||||
const stamp = Date.now();
|
||||
const sourceId = `src_realpg_${stamp}`;
|
||||
const tenantId = `t_realpg_${stamp}`;
|
||||
|
||||
const writer = svc();
|
||||
await writer.registerSource({
|
||||
source_id: sourceId,
|
||||
tenant_id: tenantId,
|
||||
project_id: "p_1",
|
||||
source_type: "api",
|
||||
name: "Live Postgres pilot source",
|
||||
access: "api",
|
||||
created_by: "realinfra-test",
|
||||
rights: {
|
||||
license_id: "official-public-data",
|
||||
allowed_purposes: ["company_enrichment"],
|
||||
pii_allowed: false,
|
||||
redistribution: "customer_deliverable",
|
||||
verified_at: "2026-09-01",
|
||||
verified_by: "realinfra-test",
|
||||
},
|
||||
provenance: {
|
||||
method: "official_api",
|
||||
authority: "Bolagsverket",
|
||||
cadence: "daily",
|
||||
freshness_sla_hours: 24,
|
||||
},
|
||||
});
|
||||
|
||||
// A BRAND NEW SourceRegistryService, own SourceService/repository
|
||||
// construction under the hood, same DATABASE_URL -- pre-migration this
|
||||
// would see NOTHING (each `new SourceService()` owned a private Map).
|
||||
const reader = svc();
|
||||
const descriptor = await reader.describeSource(sourceId, tenantId);
|
||||
expect(descriptor.source.source_id).toBe(sourceId);
|
||||
expect(descriptor.source.tenant_id).toBe(tenantId);
|
||||
expect(descriptor.rights?.license_id).toBe("official-public-data");
|
||||
expect(descriptor.rights?.allowed_purposes).toContain("company_enrichment");
|
||||
expect(descriptor.provenance?.method).toBe("official_api");
|
||||
expect(descriptor.provenance?.authority).toBe("Bolagsverket");
|
||||
|
||||
// Cross-checked with direct SQL -- real rows, not in-process bookkeeping.
|
||||
const sourceRows = await adminSql!<{ source_id: string; tenant_id: string }[]>`
|
||||
SELECT source_id, tenant_id FROM opt_sources WHERE source_id = ${sourceId}`;
|
||||
expect(sourceRows).toHaveLength(1);
|
||||
expect(sourceRows[0].tenant_id).toBe(tenantId);
|
||||
|
||||
const rightsRows = await adminSql!<{ license_id: string; allowed_purposes: string[] }[]>`
|
||||
SELECT license_id, allowed_purposes FROM opt_source_rights WHERE source_id = ${sourceId}`;
|
||||
expect(rightsRows).toHaveLength(1);
|
||||
expect(rightsRows[0].license_id).toBe("official-public-data");
|
||||
expect(rightsRows[0].allowed_purposes).toEqual(["company_enrichment"]);
|
||||
});
|
||||
|
||||
it("registering the same source_id twice -> 'already exists', enforced by real Postgres (not app-side memory)", async (ctx) => {
|
||||
if (!reachable) return ctx.skip();
|
||||
const stamp = Date.now();
|
||||
const sourceId = `src_realpg_dup_${stamp}`;
|
||||
const tenantId = `t_realpg_dup_${stamp}`;
|
||||
await svc().registerSource({
|
||||
source_id: sourceId,
|
||||
tenant_id: tenantId,
|
||||
project_id: "p_1",
|
||||
source_type: "web",
|
||||
name: "First",
|
||||
});
|
||||
await expect(
|
||||
svc().registerSource({
|
||||
source_id: sourceId,
|
||||
tenant_id: tenantId,
|
||||
project_id: "p_1",
|
||||
source_type: "web",
|
||||
name: "Second",
|
||||
}),
|
||||
).rejects.toThrow(/already exists/i);
|
||||
});
|
||||
|
||||
it("cross-tenant describe -> NOT_FOUND against real Postgres (§50, no existence oracle)", async (ctx) => {
|
||||
if (!reachable) return ctx.skip();
|
||||
const stamp = Date.now();
|
||||
const sourceId = `src_realpg_tenant_${stamp}`;
|
||||
await svc().registerSource({
|
||||
source_id: sourceId,
|
||||
tenant_id: `t_a_realpg_${stamp}`,
|
||||
project_id: "p_1",
|
||||
source_type: "web",
|
||||
name: "Tenant A source",
|
||||
});
|
||||
await expect(svc().describeSource(sourceId, `t_b_realpg_${stamp}`)).rejects.toMatchObject({
|
||||
code: "NOT_FOUND",
|
||||
});
|
||||
// And a foreign-tenant query never sees it either.
|
||||
const foreign = await svc().querySources({ tenant_id: `t_b_realpg_${stamp}` });
|
||||
expect(foreign.find((s) => s.source_id === sourceId)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("purpose gate fail-closed against real Postgres: no rights row denies every purpose", async (ctx) => {
|
||||
if (!reachable) return ctx.skip();
|
||||
const stamp = Date.now();
|
||||
const sourceId = `src_realpg_norights_${stamp}`;
|
||||
const tenantId = `t_realpg_${stamp}`;
|
||||
await svc().registerSource({
|
||||
source_id: sourceId,
|
||||
tenant_id: tenantId,
|
||||
project_id: "p_1",
|
||||
source_type: "web",
|
||||
name: "No rights yet",
|
||||
});
|
||||
await expect(svc().assertPurposeAllowed(sourceId, tenantId, "anything")).rejects.toMatchObject({
|
||||
code: "AUTHORIZATION_DENIED",
|
||||
});
|
||||
});
|
||||
|
||||
it("health checks: bounded to the latest 100 rows in Postgres, consecutive-failure count survives across service instances", async (ctx) => {
|
||||
if (!reachable) return ctx.skip();
|
||||
const stamp = Date.now();
|
||||
const sourceId = `src_realpg_health_${stamp}`;
|
||||
const tenantId = `t_realpg_${stamp}`;
|
||||
await svc().registerSource({
|
||||
source_id: sourceId,
|
||||
tenant_id: tenantId,
|
||||
project_id: "p_1",
|
||||
source_type: "api",
|
||||
name: "Health test source",
|
||||
});
|
||||
|
||||
// 103 "down" checks (oldest -> newest), each via a FRESH service
|
||||
// instance, simulating separate route-handler invocations.
|
||||
for (let i = 0; i < 103; i++) {
|
||||
await svc().recordHealthCheck({
|
||||
source_id: sourceId,
|
||||
tenant_id: tenantId,
|
||||
status: "down",
|
||||
checked_at: new Date(Date.now() - (300 - i) * 1000).toISOString(),
|
||||
});
|
||||
}
|
||||
// Then 2 "up" checks, strictly more recent than every "down" above.
|
||||
for (let i = 0; i < 2; i++) {
|
||||
await svc().recordHealthCheck({
|
||||
source_id: sourceId,
|
||||
tenant_id: tenantId,
|
||||
status: "up",
|
||||
checked_at: new Date(Date.now() - (2 - i) * 500).toISOString(),
|
||||
latency_ms: 42,
|
||||
});
|
||||
}
|
||||
|
||||
// Bounded storage: never more than 100 rows for this source_id in Postgres.
|
||||
const countRows = await adminSql!<{ n: number }[]>`
|
||||
SELECT count(*)::int AS n FROM opt_source_health_checks WHERE source_id = ${sourceId}`;
|
||||
expect(countRows[0].n).toBeLessThanOrEqual(100);
|
||||
// All 103 "down" writes plus 2 "up" writes were made; some of the
|
||||
// oldest "down" rows must have been pruned.
|
||||
expect(countRows[0].n).toBeLessThan(105);
|
||||
|
||||
const health = await svc().getHealth(sourceId, tenantId);
|
||||
expect(health.status).toBe("up");
|
||||
expect(health.consecutive_failures).toBe(0);
|
||||
|
||||
const freshness = await svc().getFreshness(sourceId, tenantId);
|
||||
expect(freshness.fresh).toBe(true);
|
||||
expect(freshness.last_success_at).toBeDefined();
|
||||
});
|
||||
|
||||
it("source.query filters by purpose and redistribution against real rows in opt_source_rights", async (ctx) => {
|
||||
if (!reachable) return ctx.skip();
|
||||
const stamp = Date.now();
|
||||
const tenantId = `t_realpg_query_${stamp}`;
|
||||
const sourceA = `src_realpg_a_${stamp}`;
|
||||
const sourceB = `src_realpg_b_${stamp}`;
|
||||
await svc().registerSource({
|
||||
source_id: sourceA,
|
||||
tenant_id: tenantId,
|
||||
project_id: "p_1",
|
||||
source_type: "api",
|
||||
name: "A",
|
||||
rights: {
|
||||
license_id: "l1",
|
||||
allowed_purposes: ["company_enrichment"],
|
||||
pii_allowed: false,
|
||||
redistribution: "customer_deliverable",
|
||||
verified_at: "2026-09-01",
|
||||
verified_by: "t",
|
||||
},
|
||||
});
|
||||
await svc().registerSource({
|
||||
source_id: sourceB,
|
||||
tenant_id: tenantId,
|
||||
project_id: "p_1",
|
||||
source_type: "web",
|
||||
name: "B",
|
||||
rights: {
|
||||
license_id: "l2",
|
||||
allowed_purposes: ["compliance"],
|
||||
pii_allowed: false,
|
||||
redistribution: "internal",
|
||||
verified_at: "2026-09-01",
|
||||
verified_by: "t",
|
||||
},
|
||||
});
|
||||
|
||||
const enrichment = await svc().querySources({ tenant_id: tenantId, purpose: "company_enrichment" });
|
||||
expect(enrichment.map((s) => s.source_id)).toEqual([sourceA]);
|
||||
|
||||
const internalOnly = await svc().querySources({ tenant_id: tenantId, redistribution: "internal" });
|
||||
expect(internalOnly.map((s) => s.source_id)).toEqual([sourceB]);
|
||||
});
|
||||
|
||||
it("setRights/setProvenance upsert in place (matching the Map.set replace semantics they replace)", async (ctx) => {
|
||||
if (!reachable) return ctx.skip();
|
||||
const stamp = Date.now();
|
||||
const sourceId = `src_realpg_upsert_${stamp}`;
|
||||
const tenantId = `t_realpg_${stamp}`;
|
||||
const registry = svc();
|
||||
await registry.registerSource({
|
||||
source_id: sourceId,
|
||||
tenant_id: tenantId,
|
||||
project_id: "p_1",
|
||||
source_type: "api",
|
||||
name: "Upsert test",
|
||||
});
|
||||
|
||||
await registry.setRights({
|
||||
source_id: sourceId,
|
||||
tenant_id: tenantId,
|
||||
license_id: "v1",
|
||||
allowed_purposes: ["compliance"],
|
||||
pii_allowed: false,
|
||||
redistribution: "internal",
|
||||
verified_at: "2026-09-01",
|
||||
verified_by: "t",
|
||||
});
|
||||
await registry.setRights({
|
||||
source_id: sourceId,
|
||||
tenant_id: tenantId,
|
||||
license_id: "v2",
|
||||
allowed_purposes: ["compliance", "company_enrichment"],
|
||||
pii_allowed: false,
|
||||
redistribution: "customer_deliverable",
|
||||
verified_at: "2026-09-02",
|
||||
verified_by: "t2",
|
||||
});
|
||||
|
||||
const rows = await adminSql!<
|
||||
{ license_id: string }[]
|
||||
>`SELECT license_id FROM opt_source_rights WHERE source_id = ${sourceId}`;
|
||||
expect(rows).toHaveLength(1); // one row, updated in place -- not two
|
||||
expect(rows[0].license_id).toBe("v2");
|
||||
|
||||
const rights = await registry.getRights(sourceId, tenantId);
|
||||
expect(rights?.license_id).toBe("v2");
|
||||
expect(rights?.allowed_purposes).toEqual(["compliance", "company_enrichment"]);
|
||||
});
|
||||
});
|
||||
@@ -75,9 +75,11 @@ describe("SourceRegistryService — purpose gate (fail closed)", () => {
|
||||
it("denies a purpose outside the allowlist", async () => {
|
||||
const registry = svc();
|
||||
await registerBase(registry);
|
||||
await expect(registry.assertPurposeAllowed("src_bolagsverket", "t_a", "marketing")).rejects.toMatchObject({
|
||||
code: "AUTHORIZATION_DENIED",
|
||||
});
|
||||
await expect(registry.assertPurposeAllowed("src_bolagsverket", "t_a", "marketing")).rejects.toMatchObject(
|
||||
{
|
||||
code: "AUTHORIZATION_DENIED",
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("denies EVERY purpose when no rights record exists (fail closed)", async () => {
|
||||
@@ -129,7 +131,7 @@ describe("SourceRegistryService — health + freshness", () => {
|
||||
await registerBase(registry);
|
||||
const old = new Date(Date.now() - 72 * 3_600_000).toISOString(); // 72h > 48h SLA
|
||||
await registry.recordSuccess("src_bolagsverket", "t_a", old);
|
||||
const freshness = registry.getFreshness("src_bolagsverket");
|
||||
const freshness = await registry.getFreshness("src_bolagsverket", "t_a");
|
||||
expect(freshness.fresh).toBe(false);
|
||||
expect(freshness.age_hours).toBeGreaterThan(48);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user