Files
0pt/tests/p0/ingestion-plane-postgres.realinfra.test.ts
T

311 lines
13 KiB
TypeScript

/**
* REAL Postgres proof for P060 DoD #1 ("a restart preserves every source")
* and for the durability the rest of the plane depends on (P062 checkpoints,
* P063 schema versions, P064 quarantine, P067 lineage, P068 ledger).
*
* Same honest-degrade convention as tests/p0/source-registry-postgres.realinfra.test.ts:
* gated on an env var, SKIPS HONESTLY when unset, never fabricates success.
*
* docker run -d --name opt-ingestion-pgtest \
* -e POSTGRES_USER=opt -e POSTGRES_PASSWORD=opt_test_pw -e POSTGRES_DB=opt_test \
* -p 15434:5432 postgres:16-alpine
* INGESTION_TEST_DATABASE_URL=postgres://opt:opt_test_pw@localhost:15434/opt_test \
* pnpm vitest run tests/p0/ingestion-plane-postgres.realinfra.test.ts
*
* "RESTART" here is modelled the only way a test can model it without killing
* the runner: every assertion after a write is made through a FRESHLY
* CONSTRUCTED service/repository over a NEW connection, so nothing can be
* answered out of an in-process Map. The stronger proof — stopping and
* starting the Postgres CONTAINER itself and re-reading — was run manually
* during this pass and is recorded in STATUS.md; it is not automated here
* because a test that restarts its own database would need docker access
* from inside CI, which these runners do not grant.
*/
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import fs from "node:fs";
import path from "node:path";
import postgres from "postgres";
import { SourceRegistryService } from "@/src/services/source-registry-service";
import { SourceService } from "@/src/services/source-service";
import { PostgresSourceRepository } from "@/src/repositories/postgres-source-repository";
import { PostgresSourceRegistryRepository } from "@/src/repositories/postgres-source-registry-repository";
import { PostgresIngestionRepository } from "@/src/repositories/postgres-ingestion-repository";
import { SchemaRegistry } from "@/src/schema/schema-registry";
import { LineageStore } from "@/src/lineage/lineage-store";
import { IngestionRunner } from "@/src/ingestion/ingestion-runner";
import { JsonlFileConnector } from "@/src/connectors/jsonl-file-connector";
import { SiaxError } from "@/src/platform/errors";
import type { RecordSchemaFields } from "@/src/schema/record-schema";
import os from "node:os";
import { promises as fsp } from "node:fs";
const TEST_DATABASE_URL = process.env.INGESTION_TEST_DATABASE_URL;
const run = TEST_DATABASE_URL ? describe : describe.skip;
let adminSql: ReturnType<typeof postgres> | undefined;
let reachable = false;
const tmpDir = path.join(os.tmpdir(), `opt-realinfra-${process.pid}`);
const filePath = path.join(tmpDir, "companies.jsonl");
const SCHEMA: RecordSchemaFields = {
org_number: { type: "string", required: true },
name: { type: "string", required: true },
employees: { type: "number", required: false },
};
const ROWS = Array.from({ length: 6 }, (_, i) => ({
org_number: `55611${i}-111${i}`,
name: `Realinfra ${i} AB`,
employees: (i + 1) * 10,
}));
const TENANT = "t_realinfra";
const SOURCE = "src_realinfra_1";
beforeAll(async () => {
if (!TEST_DATABASE_URL) {
console.warn(
"[realinfra] INGESTION_TEST_DATABASE_URL not set this run -- skipping honestly, not fabricating success.",
);
return;
}
adminSql = postgres(TEST_DATABASE_URL, { prepare: false, ssl: false, max: 2 });
await Promise.race([
adminSql`SELECT 1`,
new Promise((_, reject) => setTimeout(() => reject(new Error("connect timeout")), 5000)),
]);
reachable = true;
// Only the two migrations this plane needs. sql/001_init.sql is skipped on
// purpose: it requires the `vector` extension (pgvector), which a plain
// postgres:16-alpine does not ship, and none of the tables under test
// depend on it. Same choice tests/p0/source-registry-postgres.realinfra.test.ts
// already makes.
for (const file of ["002_source_registry.sql", "007_ingestion_plane.sql"]) {
const p = path.join(process.cwd(), "sql", file);
if (!fs.existsSync(p)) continue;
await adminSql.unsafe(fs.readFileSync(p, "utf8"));
}
// Clean slate for this suite's ids only — never a blanket TRUNCATE.
await adminSql`DELETE FROM opt_lineage_edges WHERE tenant_id = ${TENANT}`;
await adminSql`DELETE FROM opt_quality_quarantine WHERE tenant_id = ${TENANT}`;
await adminSql`DELETE FROM opt_ingestion_ledger WHERE tenant_id = ${TENANT}`;
await adminSql`DELETE FROM opt_ingestion_checkpoints WHERE tenant_id = ${TENANT}`;
await adminSql`DELETE FROM opt_source_schemas WHERE tenant_id = ${TENANT}`;
await adminSql`DELETE FROM opt_source_authorization WHERE tenant_id = ${TENANT}`;
await adminSql`DELETE FROM opt_source_schedule WHERE tenant_id = ${TENANT}`;
await adminSql`DELETE FROM opt_source_rights WHERE tenant_id = ${TENANT}`;
await adminSql`DELETE FROM opt_source_provenance WHERE tenant_id = ${TENANT}`;
await adminSql`DELETE FROM opt_source_freshness WHERE tenant_id = ${TENANT}`;
await adminSql`DELETE FROM opt_source_health_checks WHERE tenant_id = ${TENANT}`;
await adminSql`DELETE FROM opt_sources WHERE tenant_id = ${TENANT}`;
await fsp.mkdir(tmpDir, { recursive: true });
await fsp.writeFile(filePath, ROWS.map((r) => JSON.stringify(r)).join("\n") + "\n", "utf8");
});
afterAll(async () => {
await adminSql?.end({ timeout: 5 });
await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => undefined);
});
/** A brand-new service over a brand-new connection — the "restarted process". */
function freshRegistry(): SourceRegistryService {
const sql = postgres(TEST_DATABASE_URL as string, { prepare: false, ssl: false, max: 2 });
return new SourceRegistryService({
repository: new PostgresSourceRegistryRepository(sql),
sourceService: new SourceService({ repository: new PostgresSourceRepository(sql) }),
});
}
function freshIngestion(): {
repo: PostgresIngestionRepository;
runner: IngestionRunner;
lineage: LineageStore;
} {
const sql = postgres(TEST_DATABASE_URL as string, { prepare: false, ssl: false, max: 2 });
const repo = new PostgresIngestionRepository(sql);
const lineage = new LineageStore(repo);
return { repo, runner: new IngestionRunner(repo, new SchemaRegistry(repo), lineage), lineage };
}
run("P060 real Postgres — a restart preserves every source and every registry fact", () => {
it("is reachable (otherwise the rest of this suite proves nothing)", () => {
expect(reachable).toBe(true);
});
it("registers pending, refuses activation unclassified, then activates — all surviving a fresh process", async () => {
const writer = freshRegistry();
await writer.registerSource({
source_id: SOURCE,
tenant_id: TENANT,
project_id: "p_realinfra",
source_type: "file",
name: "Realinfra file drop",
owner_id: "usr_simon",
schedule: { mode: "batch", interval_seconds: 900, enabled: true },
});
// A DIFFERENT service instance over a DIFFERENT connection.
const reader = freshRegistry();
expect((await reader.sources.getSource(SOURCE)).status).toBe("pending");
await expect(reader.activateSource(SOURCE, TENANT)).rejects.toBeInstanceOf(SiaxError);
const classifier = freshRegistry();
await classifier.setRights({
source_id: SOURCE,
tenant_id: TENANT,
license_id: "proprietary:realinfra",
allowed_purposes: ["catalog"],
pii_allowed: false,
redistribution: "internal",
verified_at: "2026-09-04",
verified_by: "usr_simon",
});
await classifier.setProvenance({
source_id: SOURCE,
tenant_id: TENANT,
method: "bulk_download",
updated_at: new Date().toISOString(),
});
await classifier.setAuthorization({
source_id: SOURCE,
tenant_id: TENANT,
basis: "contract",
granted_by: "usr_simon",
granted_at: new Date().toISOString(),
});
const activator = freshRegistry();
expect((await activator.classifySource(SOURCE, TENANT)).classified).toBe(true);
expect((await activator.activateSource(SOURCE, TENANT, "usr_simon")).status).toBe("active");
// Read back through yet another fresh process AND directly in SQL, so the
// assertion cannot be satisfied by any in-process bookkeeping.
const verifier = freshRegistry();
const descriptor = await verifier.describeSource(SOURCE, TENANT);
expect(descriptor.source.status).toBe("active");
expect(descriptor.source.owner_id).toBe("usr_simon");
expect(descriptor.authorization?.basis).toBe("contract");
expect(descriptor.schedule?.interval_seconds).toBe(900);
expect(descriptor.classification?.classified).toBe(true);
const rows = await (adminSql as ReturnType<typeof postgres>)<
{ status: string; owner_id: string | null }[]
>`SELECT status, owner_id FROM opt_sources WHERE source_id = ${SOURCE}`;
expect(rows[0]).toEqual({ status: "active", owner_id: "usr_simon" });
});
it("a raw INSERT that omits status lands PENDING — the gate is in the schema too", async () => {
const sql = adminSql as ReturnType<typeof postgres>;
await sql`
INSERT INTO opt_sources (source_id, tenant_id, project_id, source_type, name)
VALUES ('src_realinfra_raw', ${TENANT}, 'p_realinfra', 'api', 'Raw insert')
ON CONFLICT (source_id) DO NOTHING`;
const rows = await sql<{ status: string }[]>`
SELECT status FROM opt_sources WHERE source_id = 'src_realinfra_raw'`;
expect(rows[0].status).toBe("pending");
});
});
run("P062/P063/P064/P067/P068 real Postgres — the durable ingestion plane", () => {
it("runs, crashes, and resumes across fresh processes with no gap and no duplicate", async () => {
const first = freshIngestion();
const delivered: string[] = [];
let pages = 0;
const input = {
source_id: SOURCE,
tenant_id: TENANT,
project_id: "p_realinfra",
mode: "batch" as const,
connector: new JsonlFileConnector(),
config: { file_path: filePath, partition_key: "companies", schema: SCHEMA, record_key_field: "org_number" },
page_size: 2,
output_ref: "0pt:dataset:ds_realinfra",
};
await expect(
first.runner.run({
...input,
sink: async (batch) => {
pages++;
if (pages === 2) throw new Error("simulated process death");
for (const e of batch) delivered.push(e.record_key);
},
}),
).rejects.toThrow(/simulated process death/);
expect(delivered).toHaveLength(2);
// Fresh everything — only Postgres carries the position forward.
const second = freshIngestion();
const report = await second.runner.run({
...input,
connector: new JsonlFileConnector(),
sink: async (batch) => {
for (const e of batch) delivered.push(e.record_key);
},
});
expect(report.status).toBe("succeeded");
expect(delivered).toHaveLength(ROWS.length);
expect(new Set(delivered).size).toBe(ROWS.length);
const cp = await second.repo.getCheckpoint(SOURCE, "companies");
expect(cp?.records_committed).toBe(ROWS.length);
});
it("the schema version, ledger, and lineage are all readable from a fresh process", async () => {
const fresh = freshIngestion();
expect((await new SchemaRegistry(fresh.repo).getLatest(SOURCE))?.schema_version).toBe(1);
expect(await fresh.repo.countLedger(SOURCE)).toBe(ROWS.length);
const upstream = await fresh.lineage.upstreamOf(TENANT, "0pt:dataset:ds_realinfra");
expect(upstream.length).toBeGreaterThanOrEqual(1);
expect(upstream[0].input_ref).toBe(`0pt:source:${SOURCE}`);
});
it("a repeated backfill is idempotent against the durable ledger", async () => {
const fresh = freshIngestion();
const emitted: string[] = [];
const report = await fresh.runner.backfill({
source_id: SOURCE,
tenant_id: TENANT,
project_id: "p_realinfra",
mode: "batch",
connector: new JsonlFileConnector(),
config: { file_path: filePath, partition_key: "companies", schema: SCHEMA, record_key_field: "org_number" },
sink: async (batch) => {
for (const e of batch) emitted.push(e.record_key);
},
page_size: 100,
range: { partition_key: "companies", from: "0", to: String(ROWS.length) },
});
expect(report.envelopes_emitted).toBe(0);
expect(report.duplicates_skipped).toBe(ROWS.length);
expect(emitted).toHaveLength(0);
});
it("quarantine survives to a fresh process with the full envelope", async () => {
const fresh = freshIngestion();
await fresh.runner.backfill({
source_id: SOURCE,
tenant_id: TENANT,
project_id: "p_realinfra",
mode: "batch",
connector: new JsonlFileConnector(),
config: { file_path: filePath, partition_key: "companies", schema: SCHEMA, record_key_field: "org_number" },
sink: async () => undefined,
page_size: 100,
quality_policy: {
source_id: SOURCE,
tenant_id: TENANT,
expectations: [
{ expectation_type: "expect_column_values_to_be_between", kwargs: { column: "employees", min_value: 0, max_value: 20 } },
],
on_violation: "quarantine",
},
range: { partition_key: "companies", from: "0", to: String(ROWS.length) },
});
const reader = freshIngestion();
const quarantined = await reader.repo.listQuarantine(SOURCE);
expect(quarantined.length).toBe(ROWS.length - 2);
expect(quarantined[0].envelope.record_key).toBeTruthy();
expect(quarantined[0].violations[0].expectation_type).toBe("expect_column_values_to_be_between");
});
});