/** * REAL Postgres integration test (Master-DoD §48/§50) for the Source * Registry's new persistence layer (src/repositories/*, * sql/002_source_registry.sql) -- the P010 task's central deliverable: this * is what proves the Map -> Postgres migration actually works, not just * that it typechecks. * * Same honest-degrade convention as tests/p0/iceberg-write-service.realinfra.test.ts * and tests/integration/analytics-clickhouse.integration.test.ts: gated on * an env var (SOURCE_REGISTRY_TEST_DATABASE_URL), SKIPS HONESTLY when unset, * never fabricates success. Point it at a disposable Postgres, e.g.: * * docker run -d --name opt-source-registry-pgtest \ * -e POSTGRES_USER=opt -e POSTGRES_PASSWORD=opt_test_pw -e POSTGRES_DB=opt_test \ * -p 15433:5432 postgres:16-alpine * SOURCE_REGISTRY_TEST_DATABASE_URL=postgres://opt:opt_test_pw@localhost:15433/opt_test \ * pnpm vitest run tests/p0/source-registry-postgres.realinfra.test.ts * * This is NOT a mock-repository test -- every assertion below is driven * through SourceRegistryService with `databaseUrl` set, i.e. the real * PostgresSourceRepository/PostgresSourceRegistryRepository HTTP-free SQL * path (postgres.js against the live container), with a handful of * assertions cross-checked with direct SQL against the same database to * prove the rows are really there, not an artifact of the service's own * in-process bookkeeping. * * beforeAll applies every sql/NNN_*.sql file in order (same restore-drill * pattern as tests/analytics/analytics-domain-postgres.realinfra.test.ts), * not just sql/002_source_registry.sql -- this test does not require the * operator to have pre-run migrations. * * CORRECTED (this pass, 2026-09-14): it used to apply ONLY * sql/002_source_registry.sql. That was enough the day this file was * written, but sql/007_ingestion_plane.sql later added `opt_sources.owner_id` * (`ALTER TABLE opt_sources ADD COLUMN IF NOT EXISTS owner_id TEXT`), and * src/repositories/postgres-source-repository.ts's INSERT has named that * column ever since. Because this env var is never set in CI (confirmed by * grep over .gitea/workflows/ci.yml — no job references * SOURCE_REGISTRY_TEST_DATABASE_URL), the drift was never caught: every * assertion below failed with `column "owner_id" of relation "opt_sources" * does not exist` the moment it was actually run against a real, freshly * migrated-to-002-only Postgres. The fix is to stop hand-picking one * migration file and replay the whole chain, exactly like the analytics * suite already does. */ import { afterAll, beforeAll, describe, expect, it } from "vitest"; import fs from "node:fs"; import path from "node:path"; import postgres from "postgres"; import { SourceRegistryService } from "@/src/services/source-registry-service"; import { closeRepositorySqlForTests } from "@/src/repositories/db"; const TEST_DATABASE_URL = process.env.SOURCE_REGISTRY_TEST_DATABASE_URL; const run = TEST_DATABASE_URL ? describe : describe.skip; let adminSql: ReturnType | undefined; let reachable = false; let unreachableReason = ""; beforeAll(async () => { if (!TEST_DATABASE_URL) { console.warn( "[realinfra] SOURCE_REGISTRY_TEST_DATABASE_URL not set this run -- skipping honestly, not fabricating success.", ); return; } try { adminSql = postgres(TEST_DATABASE_URL, { prepare: false, ssl: false, max: 2 }); await Promise.race([ adminSql`SELECT 1`, new Promise((_, reject) => setTimeout(() => reject(new Error("connect timeout")), 5000)), ]); // Replay every migration in order, not just 002 -- see the file header // "CORRECTED" note. Tolerate an individual file failing (mirrors // analytics-domain-postgres.realinfra.test.ts: sql/001 needs the // `vector` extension, which a stock postgres:16-alpine does not carry) // as long as the files this test actually depends on (002, 007) apply. const dir = path.join(process.cwd(), "sql"); for (const file of fs .readdirSync(dir) .filter((f) => f.endsWith(".sql")) .sort()) { const body = fs.readFileSync(path.join(dir, file), "utf8"); try { await adminSql.unsafe(body); } catch (err) { console.warn(`[realinfra] ${file} did not apply cleanly: ${String(err).slice(0, 200)}`); } } reachable = true; } catch (err) { reachable = false; unreachableReason = err instanceof Error ? err.message : String(err); } }, 30000); afterAll(async () => { await adminSql?.end({ timeout: 5 }); await closeRepositorySqlForTests(); }); function svc(): SourceRegistryService { return new SourceRegistryService({ databaseUrl: TEST_DATABASE_URL }); } run("SourceRegistryService against a LIVE Postgres (sql/002_source_registry.sql)", () => { it("persists across independent service instances -- the exact bug this migration fixes (in-memory Maps were per-instance, not even per-process)", async (ctx) => { if (!reachable) { console.warn( `[realinfra] Postgres not reachable this run (${unreachableReason}) -- skipping honestly.`, ); ctx.skip(); return; } const stamp = Date.now(); const sourceId = `src_realpg_${stamp}`; const tenantId = `t_realpg_${stamp}`; const writer = svc(); await writer.registerSource({ source_id: sourceId, tenant_id: tenantId, project_id: "p_1", source_type: "api", name: "Live Postgres pilot source", access: "api", created_by: "realinfra-test", rights: { license_id: "official-public-data", allowed_purposes: ["company_enrichment"], pii_allowed: false, redistribution: "customer_deliverable", verified_at: "2026-09-01", verified_by: "realinfra-test", }, provenance: { method: "official_api", authority: "Bolagsverket", cadence: "daily", freshness_sla_hours: 24, }, }); // A BRAND NEW SourceRegistryService, own SourceService/repository // construction under the hood, same DATABASE_URL -- pre-migration this // would see NOTHING (each `new SourceService()` owned a private Map). const reader = svc(); const descriptor = await reader.describeSource(sourceId, tenantId); expect(descriptor.source.source_id).toBe(sourceId); expect(descriptor.source.tenant_id).toBe(tenantId); expect(descriptor.rights?.license_id).toBe("official-public-data"); expect(descriptor.rights?.allowed_purposes).toContain("company_enrichment"); expect(descriptor.provenance?.method).toBe("official_api"); expect(descriptor.provenance?.authority).toBe("Bolagsverket"); // Cross-checked with direct SQL -- real rows, not in-process bookkeeping. const sourceRows = await adminSql!<{ source_id: string; tenant_id: string }[]>` SELECT source_id, tenant_id FROM opt_sources WHERE source_id = ${sourceId}`; expect(sourceRows).toHaveLength(1); expect(sourceRows[0].tenant_id).toBe(tenantId); const rightsRows = await adminSql!<{ license_id: string; allowed_purposes: string[] }[]>` SELECT license_id, allowed_purposes FROM opt_source_rights WHERE source_id = ${sourceId}`; expect(rightsRows).toHaveLength(1); expect(rightsRows[0].license_id).toBe("official-public-data"); expect(rightsRows[0].allowed_purposes).toEqual(["company_enrichment"]); }); it("registering the same source_id twice -> 'already exists', enforced by real Postgres (not app-side memory)", async (ctx) => { if (!reachable) return ctx.skip(); const stamp = Date.now(); const sourceId = `src_realpg_dup_${stamp}`; const tenantId = `t_realpg_dup_${stamp}`; await svc().registerSource({ source_id: sourceId, tenant_id: tenantId, project_id: "p_1", source_type: "web", name: "First", }); await expect( svc().registerSource({ source_id: sourceId, tenant_id: tenantId, project_id: "p_1", source_type: "web", name: "Second", }), ).rejects.toThrow(/already exists/i); }); it("cross-tenant describe -> NOT_FOUND against real Postgres (§50, no existence oracle)", async (ctx) => { if (!reachable) return ctx.skip(); const stamp = Date.now(); const sourceId = `src_realpg_tenant_${stamp}`; await svc().registerSource({ source_id: sourceId, tenant_id: `t_a_realpg_${stamp}`, project_id: "p_1", source_type: "web", name: "Tenant A source", }); await expect(svc().describeSource(sourceId, `t_b_realpg_${stamp}`)).rejects.toMatchObject({ code: "NOT_FOUND", }); // And a foreign-tenant query never sees it either. const foreign = await svc().querySources({ tenant_id: `t_b_realpg_${stamp}` }); expect(foreign.find((s) => s.source_id === sourceId)).toBeUndefined(); }); it("purpose gate fail-closed against real Postgres: no rights row denies every purpose", async (ctx) => { if (!reachable) return ctx.skip(); const stamp = Date.now(); const sourceId = `src_realpg_norights_${stamp}`; const tenantId = `t_realpg_${stamp}`; await svc().registerSource({ source_id: sourceId, tenant_id: tenantId, project_id: "p_1", source_type: "web", name: "No rights yet", }); await expect(svc().assertPurposeAllowed(sourceId, tenantId, "anything")).rejects.toMatchObject({ code: "AUTHORIZATION_DENIED", }); }); it("health checks: bounded to the latest 100 rows in Postgres, consecutive-failure count survives across service instances", async (ctx) => { if (!reachable) return ctx.skip(); const stamp = Date.now(); const sourceId = `src_realpg_health_${stamp}`; const tenantId = `t_realpg_${stamp}`; await svc().registerSource({ source_id: sourceId, tenant_id: tenantId, project_id: "p_1", source_type: "api", name: "Health test source", }); // 103 "down" checks (oldest -> newest), each via a FRESH service // instance, simulating separate route-handler invocations. for (let i = 0; i < 103; i++) { await svc().recordHealthCheck({ source_id: sourceId, tenant_id: tenantId, status: "down", checked_at: new Date(Date.now() - (300 - i) * 1000).toISOString(), }); } // Then 2 "up" checks, strictly more recent than every "down" above. for (let i = 0; i < 2; i++) { await svc().recordHealthCheck({ source_id: sourceId, tenant_id: tenantId, status: "up", checked_at: new Date(Date.now() - (2 - i) * 500).toISOString(), latency_ms: 42, }); } // Bounded storage: never more than 100 rows for this source_id in Postgres. const countRows = await adminSql!<{ n: number }[]>` SELECT count(*)::int AS n FROM opt_source_health_checks WHERE source_id = ${sourceId}`; expect(countRows[0].n).toBeLessThanOrEqual(100); // All 103 "down" writes plus 2 "up" writes were made; some of the // oldest "down" rows must have been pruned. expect(countRows[0].n).toBeLessThan(105); const health = await svc().getHealth(sourceId, tenantId); expect(health.status).toBe("up"); expect(health.consecutive_failures).toBe(0); const freshness = await svc().getFreshness(sourceId, tenantId); expect(freshness.fresh).toBe(true); expect(freshness.last_success_at).toBeDefined(); }); it("source.query filters by purpose and redistribution against real rows in opt_source_rights", async (ctx) => { if (!reachable) return ctx.skip(); const stamp = Date.now(); const tenantId = `t_realpg_query_${stamp}`; const sourceA = `src_realpg_a_${stamp}`; const sourceB = `src_realpg_b_${stamp}`; await svc().registerSource({ source_id: sourceA, tenant_id: tenantId, project_id: "p_1", source_type: "api", name: "A", rights: { license_id: "l1", allowed_purposes: ["company_enrichment"], pii_allowed: false, redistribution: "customer_deliverable", verified_at: "2026-09-01", verified_by: "t", }, }); await svc().registerSource({ source_id: sourceB, tenant_id: tenantId, project_id: "p_1", source_type: "web", name: "B", rights: { license_id: "l2", allowed_purposes: ["compliance"], pii_allowed: false, redistribution: "internal", verified_at: "2026-09-01", verified_by: "t", }, }); const enrichment = await svc().querySources({ tenant_id: tenantId, purpose: "company_enrichment" }); expect(enrichment.map((s) => s.source_id)).toEqual([sourceA]); const internalOnly = await svc().querySources({ tenant_id: tenantId, redistribution: "internal" }); expect(internalOnly.map((s) => s.source_id)).toEqual([sourceB]); }); it("setRights/setProvenance upsert in place (matching the Map.set replace semantics they replace)", async (ctx) => { if (!reachable) return ctx.skip(); const stamp = Date.now(); const sourceId = `src_realpg_upsert_${stamp}`; const tenantId = `t_realpg_${stamp}`; const registry = svc(); await registry.registerSource({ source_id: sourceId, tenant_id: tenantId, project_id: "p_1", source_type: "api", name: "Upsert test", }); await registry.setRights({ source_id: sourceId, tenant_id: tenantId, license_id: "v1", allowed_purposes: ["compliance"], pii_allowed: false, redistribution: "internal", verified_at: "2026-09-01", verified_by: "t", }); await registry.setRights({ source_id: sourceId, tenant_id: tenantId, license_id: "v2", allowed_purposes: ["compliance", "company_enrichment"], pii_allowed: false, redistribution: "customer_deliverable", verified_at: "2026-09-02", verified_by: "t2", }); const rows = await adminSql!< { license_id: string }[] >`SELECT license_id FROM opt_source_rights WHERE source_id = ${sourceId}`; expect(rows).toHaveLength(1); // one row, updated in place -- not two expect(rows[0].license_id).toBe("v2"); const rights = await registry.getRights(sourceId, tenantId); expect(rights?.license_id).toBe("v2"); expect(rights?.allowed_purposes).toEqual(["compliance", "company_enrichment"]); }); });