/** * REAL Postgres proof for P060 DoD #1 ("a restart preserves every source") * and for the durability the rest of the plane depends on (P062 checkpoints, * P063 schema versions, P064 quarantine, P067 lineage, P068 ledger). * * Same honest-degrade convention as tests/p0/source-registry-postgres.realinfra.test.ts: * gated on an env var, SKIPS HONESTLY when unset, never fabricates success. * * docker run -d --name opt-ingestion-pgtest \ * -e POSTGRES_USER=opt -e POSTGRES_PASSWORD=opt_test_pw -e POSTGRES_DB=opt_test \ * -p 15434:5432 postgres:16-alpine * INGESTION_TEST_DATABASE_URL=postgres://opt:opt_test_pw@localhost:15434/opt_test \ * pnpm vitest run tests/p0/ingestion-plane-postgres.realinfra.test.ts * * "RESTART" here is modelled the only way a test can model it without killing * the runner: every assertion after a write is made through a FRESHLY * CONSTRUCTED service/repository over a NEW connection, so nothing can be * answered out of an in-process Map. The stronger proof — stopping and * starting the Postgres CONTAINER itself and re-reading — was run manually * during this pass and is recorded in STATUS.md; it is not automated here * because a test that restarts its own database would need docker access * from inside CI, which these runners do not grant. */ import { afterAll, beforeAll, describe, expect, it } from "vitest"; import fs from "node:fs"; import path from "node:path"; import postgres from "postgres"; import { SourceRegistryService } from "@/src/services/source-registry-service"; import { SourceService } from "@/src/services/source-service"; import { PostgresSourceRepository } from "@/src/repositories/postgres-source-repository"; import { PostgresSourceRegistryRepository } from "@/src/repositories/postgres-source-registry-repository"; import { PostgresIngestionRepository } from "@/src/repositories/postgres-ingestion-repository"; import { SchemaRegistry } from "@/src/schema/schema-registry"; import { LineageStore } from "@/src/lineage/lineage-store"; import { IngestionRunner } from "@/src/ingestion/ingestion-runner"; import { JsonlFileConnector } from "@/src/connectors/jsonl-file-connector"; import { SiaxError } from "@/src/platform/errors"; import type { RecordSchemaFields } from "@/src/schema/record-schema"; import os from "node:os"; import { promises as fsp } from "node:fs"; const TEST_DATABASE_URL = process.env.INGESTION_TEST_DATABASE_URL; const run = TEST_DATABASE_URL ? describe : describe.skip; let adminSql: ReturnType | undefined; let reachable = false; const tmpDir = path.join(os.tmpdir(), `opt-realinfra-${process.pid}`); const filePath = path.join(tmpDir, "companies.jsonl"); const SCHEMA: RecordSchemaFields = { org_number: { type: "string", required: true }, name: { type: "string", required: true }, employees: { type: "number", required: false }, }; const ROWS = Array.from({ length: 6 }, (_, i) => ({ org_number: `55611${i}-111${i}`, name: `Realinfra ${i} AB`, employees: (i + 1) * 10, })); const TENANT = "t_realinfra"; const SOURCE = "src_realinfra_1"; beforeAll(async () => { if (!TEST_DATABASE_URL) { console.warn( "[realinfra] INGESTION_TEST_DATABASE_URL not set this run -- skipping honestly, not fabricating success.", ); return; } adminSql = postgres(TEST_DATABASE_URL, { prepare: false, ssl: false, max: 2 }); await Promise.race([ adminSql`SELECT 1`, new Promise((_, reject) => setTimeout(() => reject(new Error("connect timeout")), 5000)), ]); reachable = true; // Only the two migrations this plane needs. sql/001_init.sql is skipped on // purpose: it requires the `vector` extension (pgvector), which a plain // postgres:16-alpine does not ship, and none of the tables under test // depend on it. Same choice tests/p0/source-registry-postgres.realinfra.test.ts // already makes. for (const file of ["002_source_registry.sql", "007_ingestion_plane.sql"]) { const p = path.join(process.cwd(), "sql", file); if (!fs.existsSync(p)) continue; await adminSql.unsafe(fs.readFileSync(p, "utf8")); } // Clean slate for this suite's ids only — never a blanket TRUNCATE. await adminSql`DELETE FROM opt_lineage_edges WHERE tenant_id = ${TENANT}`; await adminSql`DELETE FROM opt_quality_quarantine WHERE tenant_id = ${TENANT}`; await adminSql`DELETE FROM opt_ingestion_ledger WHERE tenant_id = ${TENANT}`; await adminSql`DELETE FROM opt_ingestion_checkpoints WHERE tenant_id = ${TENANT}`; await adminSql`DELETE FROM opt_source_schemas WHERE tenant_id = ${TENANT}`; await adminSql`DELETE FROM opt_source_authorization WHERE tenant_id = ${TENANT}`; await adminSql`DELETE FROM opt_source_schedule WHERE tenant_id = ${TENANT}`; await adminSql`DELETE FROM opt_source_rights WHERE tenant_id = ${TENANT}`; await adminSql`DELETE FROM opt_source_provenance WHERE tenant_id = ${TENANT}`; await adminSql`DELETE FROM opt_source_freshness WHERE tenant_id = ${TENANT}`; await adminSql`DELETE FROM opt_source_health_checks WHERE tenant_id = ${TENANT}`; await adminSql`DELETE FROM opt_sources WHERE tenant_id = ${TENANT}`; await fsp.mkdir(tmpDir, { recursive: true }); await fsp.writeFile(filePath, ROWS.map((r) => JSON.stringify(r)).join("\n") + "\n", "utf8"); }); afterAll(async () => { await adminSql?.end({ timeout: 5 }); await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => undefined); }); /** A brand-new service over a brand-new connection — the "restarted process". */ function freshRegistry(): SourceRegistryService { const sql = postgres(TEST_DATABASE_URL as string, { prepare: false, ssl: false, max: 2 }); return new SourceRegistryService({ repository: new PostgresSourceRegistryRepository(sql), sourceService: new SourceService({ repository: new PostgresSourceRepository(sql) }), }); } function freshIngestion(): { repo: PostgresIngestionRepository; runner: IngestionRunner; lineage: LineageStore; } { const sql = postgres(TEST_DATABASE_URL as string, { prepare: false, ssl: false, max: 2 }); const repo = new PostgresIngestionRepository(sql); const lineage = new LineageStore(repo); return { repo, runner: new IngestionRunner(repo, new SchemaRegistry(repo), lineage), lineage }; } run("P060 real Postgres — a restart preserves every source and every registry fact", () => { it("is reachable (otherwise the rest of this suite proves nothing)", () => { expect(reachable).toBe(true); }); it("registers pending, refuses activation unclassified, then activates — all surviving a fresh process", async () => { const writer = freshRegistry(); await writer.registerSource({ source_id: SOURCE, tenant_id: TENANT, project_id: "p_realinfra", source_type: "file", name: "Realinfra file drop", owner_id: "usr_simon", schedule: { mode: "batch", interval_seconds: 900, enabled: true }, }); // A DIFFERENT service instance over a DIFFERENT connection. const reader = freshRegistry(); expect((await reader.sources.getSource(SOURCE)).status).toBe("pending"); await expect(reader.activateSource(SOURCE, TENANT)).rejects.toBeInstanceOf(SiaxError); const classifier = freshRegistry(); await classifier.setRights({ source_id: SOURCE, tenant_id: TENANT, license_id: "proprietary:realinfra", allowed_purposes: ["catalog"], pii_allowed: false, redistribution: "internal", verified_at: "2026-09-04", verified_by: "usr_simon", }); await classifier.setProvenance({ source_id: SOURCE, tenant_id: TENANT, method: "bulk_download", updated_at: new Date().toISOString(), }); await classifier.setAuthorization({ source_id: SOURCE, tenant_id: TENANT, basis: "contract", granted_by: "usr_simon", granted_at: new Date().toISOString(), }); const activator = freshRegistry(); expect((await activator.classifySource(SOURCE, TENANT)).classified).toBe(true); expect((await activator.activateSource(SOURCE, TENANT, "usr_simon")).status).toBe("active"); // Read back through yet another fresh process AND directly in SQL, so the // assertion cannot be satisfied by any in-process bookkeeping. const verifier = freshRegistry(); const descriptor = await verifier.describeSource(SOURCE, TENANT); expect(descriptor.source.status).toBe("active"); expect(descriptor.source.owner_id).toBe("usr_simon"); expect(descriptor.authorization?.basis).toBe("contract"); expect(descriptor.schedule?.interval_seconds).toBe(900); expect(descriptor.classification?.classified).toBe(true); const rows = await (adminSql as ReturnType)< { status: string; owner_id: string | null }[] >`SELECT status, owner_id FROM opt_sources WHERE source_id = ${SOURCE}`; expect(rows[0]).toEqual({ status: "active", owner_id: "usr_simon" }); }); it("a raw INSERT that omits status lands PENDING — the gate is in the schema too", async () => { const sql = adminSql as ReturnType; await sql` INSERT INTO opt_sources (source_id, tenant_id, project_id, source_type, name) VALUES ('src_realinfra_raw', ${TENANT}, 'p_realinfra', 'api', 'Raw insert') ON CONFLICT (source_id) DO NOTHING`; const rows = await sql<{ status: string }[]>` SELECT status FROM opt_sources WHERE source_id = 'src_realinfra_raw'`; expect(rows[0].status).toBe("pending"); }); }); run("P062/P063/P064/P067/P068 real Postgres — the durable ingestion plane", () => { it("runs, crashes, and resumes across fresh processes with no gap and no duplicate", async () => { const first = freshIngestion(); const delivered: string[] = []; let pages = 0; const input = { source_id: SOURCE, tenant_id: TENANT, project_id: "p_realinfra", mode: "batch" as const, connector: new JsonlFileConnector(), config: { file_path: filePath, partition_key: "companies", schema: SCHEMA, record_key_field: "org_number" }, page_size: 2, output_ref: "0pt:dataset:ds_realinfra", }; await expect( first.runner.run({ ...input, sink: async (batch) => { pages++; if (pages === 2) throw new Error("simulated process death"); for (const e of batch) delivered.push(e.record_key); }, }), ).rejects.toThrow(/simulated process death/); expect(delivered).toHaveLength(2); // Fresh everything — only Postgres carries the position forward. const second = freshIngestion(); const report = await second.runner.run({ ...input, connector: new JsonlFileConnector(), sink: async (batch) => { for (const e of batch) delivered.push(e.record_key); }, }); expect(report.status).toBe("succeeded"); expect(delivered).toHaveLength(ROWS.length); expect(new Set(delivered).size).toBe(ROWS.length); const cp = await second.repo.getCheckpoint(SOURCE, "companies"); expect(cp?.records_committed).toBe(ROWS.length); }); it("the schema version, ledger, and lineage are all readable from a fresh process", async () => { const fresh = freshIngestion(); expect((await new SchemaRegistry(fresh.repo).getLatest(SOURCE))?.schema_version).toBe(1); expect(await fresh.repo.countLedger(SOURCE)).toBe(ROWS.length); const upstream = await fresh.lineage.upstreamOf(TENANT, "0pt:dataset:ds_realinfra"); expect(upstream.length).toBeGreaterThanOrEqual(1); expect(upstream[0].input_ref).toBe(`0pt:source:${SOURCE}`); }); it("a repeated backfill is idempotent against the durable ledger", async () => { const fresh = freshIngestion(); const emitted: string[] = []; const report = await fresh.runner.backfill({ source_id: SOURCE, tenant_id: TENANT, project_id: "p_realinfra", mode: "batch", connector: new JsonlFileConnector(), config: { file_path: filePath, partition_key: "companies", schema: SCHEMA, record_key_field: "org_number" }, sink: async (batch) => { for (const e of batch) emitted.push(e.record_key); }, page_size: 100, range: { partition_key: "companies", from: "0", to: String(ROWS.length) }, }); expect(report.envelopes_emitted).toBe(0); expect(report.duplicates_skipped).toBe(ROWS.length); expect(emitted).toHaveLength(0); }); it("quarantine survives to a fresh process with the full envelope", async () => { const fresh = freshIngestion(); await fresh.runner.backfill({ source_id: SOURCE, tenant_id: TENANT, project_id: "p_realinfra", mode: "batch", connector: new JsonlFileConnector(), config: { file_path: filePath, partition_key: "companies", schema: SCHEMA, record_key_field: "org_number" }, sink: async () => undefined, page_size: 100, quality_policy: { source_id: SOURCE, tenant_id: TENANT, expectations: [ { expectation_type: "expect_column_values_to_be_between", kwargs: { column: "employees", min_value: 0, max_value: 20 } }, ], on_violation: "quarantine", }, range: { partition_key: "companies", from: "0", to: String(ROWS.length) }, }); const reader = freshIngestion(); const quarantined = await reader.repo.listQuarantine(SOURCE); expect(quarantined.length).toBe(ROWS.length - 2); expect(quarantined[0].envelope.record_key).toBeTruthy(); expect(quarantined[0].violations[0].expectation_type).toBe("expect_column_values_to_be_between"); }); });