311 lines
13 KiB
TypeScript
311 lines
13 KiB
TypeScript
/**
|
|
* REAL Postgres proof for P060 DoD #1 ("a restart preserves every source")
|
|
* and for the durability the rest of the plane depends on (P062 checkpoints,
|
|
* P063 schema versions, P064 quarantine, P067 lineage, P068 ledger).
|
|
*
|
|
* Same honest-degrade convention as tests/p0/source-registry-postgres.realinfra.test.ts:
|
|
* gated on an env var, SKIPS HONESTLY when unset, never fabricates success.
|
|
*
|
|
* docker run -d --name opt-ingestion-pgtest \
|
|
* -e POSTGRES_USER=opt -e POSTGRES_PASSWORD=opt_test_pw -e POSTGRES_DB=opt_test \
|
|
* -p 15434:5432 postgres:16-alpine
|
|
* INGESTION_TEST_DATABASE_URL=postgres://opt:opt_test_pw@localhost:15434/opt_test \
|
|
* pnpm vitest run tests/p0/ingestion-plane-postgres.realinfra.test.ts
|
|
*
|
|
* "RESTART" here is modelled the only way a test can model it without killing
|
|
* the runner: every assertion after a write is made through a FRESHLY
|
|
* CONSTRUCTED service/repository over a NEW connection, so nothing can be
|
|
* answered out of an in-process Map. The stronger proof — stopping and
|
|
* starting the Postgres CONTAINER itself and re-reading — was run manually
|
|
* during this pass and is recorded in STATUS.md; it is not automated here
|
|
* because a test that restarts its own database would need docker access
|
|
* from inside CI, which these runners do not grant.
|
|
*/
|
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import postgres from "postgres";
|
|
import { SourceRegistryService } from "@/src/services/source-registry-service";
|
|
import { SourceService } from "@/src/services/source-service";
|
|
import { PostgresSourceRepository } from "@/src/repositories/postgres-source-repository";
|
|
import { PostgresSourceRegistryRepository } from "@/src/repositories/postgres-source-registry-repository";
|
|
import { PostgresIngestionRepository } from "@/src/repositories/postgres-ingestion-repository";
|
|
import { SchemaRegistry } from "@/src/schema/schema-registry";
|
|
import { LineageStore } from "@/src/lineage/lineage-store";
|
|
import { IngestionRunner } from "@/src/ingestion/ingestion-runner";
|
|
import { JsonlFileConnector } from "@/src/connectors/jsonl-file-connector";
|
|
import { SiaxError } from "@/src/platform/errors";
|
|
import type { RecordSchemaFields } from "@/src/schema/record-schema";
|
|
import os from "node:os";
|
|
import { promises as fsp } from "node:fs";
|
|
|
|
const TEST_DATABASE_URL = process.env.INGESTION_TEST_DATABASE_URL;
|
|
const run = TEST_DATABASE_URL ? describe : describe.skip;
|
|
|
|
let adminSql: ReturnType<typeof postgres> | undefined;
|
|
let reachable = false;
|
|
const tmpDir = path.join(os.tmpdir(), `opt-realinfra-${process.pid}`);
|
|
const filePath = path.join(tmpDir, "companies.jsonl");
|
|
|
|
const SCHEMA: RecordSchemaFields = {
|
|
org_number: { type: "string", required: true },
|
|
name: { type: "string", required: true },
|
|
employees: { type: "number", required: false },
|
|
};
|
|
const ROWS = Array.from({ length: 6 }, (_, i) => ({
|
|
org_number: `55611${i}-111${i}`,
|
|
name: `Realinfra ${i} AB`,
|
|
employees: (i + 1) * 10,
|
|
}));
|
|
|
|
const TENANT = "t_realinfra";
|
|
const SOURCE = "src_realinfra_1";
|
|
|
|
beforeAll(async () => {
|
|
if (!TEST_DATABASE_URL) {
|
|
console.warn(
|
|
"[realinfra] INGESTION_TEST_DATABASE_URL not set this run -- skipping honestly, not fabricating success.",
|
|
);
|
|
return;
|
|
}
|
|
adminSql = postgres(TEST_DATABASE_URL, { prepare: false, ssl: false, max: 2 });
|
|
await Promise.race([
|
|
adminSql`SELECT 1`,
|
|
new Promise((_, reject) => setTimeout(() => reject(new Error("connect timeout")), 5000)),
|
|
]);
|
|
reachable = true;
|
|
// Only the two migrations this plane needs. sql/001_init.sql is skipped on
|
|
// purpose: it requires the `vector` extension (pgvector), which a plain
|
|
// postgres:16-alpine does not ship, and none of the tables under test
|
|
// depend on it. Same choice tests/p0/source-registry-postgres.realinfra.test.ts
|
|
// already makes.
|
|
for (const file of ["002_source_registry.sql", "007_ingestion_plane.sql"]) {
|
|
const p = path.join(process.cwd(), "sql", file);
|
|
if (!fs.existsSync(p)) continue;
|
|
await adminSql.unsafe(fs.readFileSync(p, "utf8"));
|
|
}
|
|
// Clean slate for this suite's ids only — never a blanket TRUNCATE.
|
|
await adminSql`DELETE FROM opt_lineage_edges WHERE tenant_id = ${TENANT}`;
|
|
await adminSql`DELETE FROM opt_quality_quarantine WHERE tenant_id = ${TENANT}`;
|
|
await adminSql`DELETE FROM opt_ingestion_ledger WHERE tenant_id = ${TENANT}`;
|
|
await adminSql`DELETE FROM opt_ingestion_checkpoints WHERE tenant_id = ${TENANT}`;
|
|
await adminSql`DELETE FROM opt_source_schemas WHERE tenant_id = ${TENANT}`;
|
|
await adminSql`DELETE FROM opt_source_authorization WHERE tenant_id = ${TENANT}`;
|
|
await adminSql`DELETE FROM opt_source_schedule WHERE tenant_id = ${TENANT}`;
|
|
await adminSql`DELETE FROM opt_source_rights WHERE tenant_id = ${TENANT}`;
|
|
await adminSql`DELETE FROM opt_source_provenance WHERE tenant_id = ${TENANT}`;
|
|
await adminSql`DELETE FROM opt_source_freshness WHERE tenant_id = ${TENANT}`;
|
|
await adminSql`DELETE FROM opt_source_health_checks WHERE tenant_id = ${TENANT}`;
|
|
await adminSql`DELETE FROM opt_sources WHERE tenant_id = ${TENANT}`;
|
|
|
|
await fsp.mkdir(tmpDir, { recursive: true });
|
|
await fsp.writeFile(filePath, ROWS.map((r) => JSON.stringify(r)).join("\n") + "\n", "utf8");
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await adminSql?.end({ timeout: 5 });
|
|
await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => undefined);
|
|
});
|
|
|
|
/** A brand-new service over a brand-new connection — the "restarted process". */
|
|
function freshRegistry(): SourceRegistryService {
|
|
const sql = postgres(TEST_DATABASE_URL as string, { prepare: false, ssl: false, max: 2 });
|
|
return new SourceRegistryService({
|
|
repository: new PostgresSourceRegistryRepository(sql),
|
|
sourceService: new SourceService({ repository: new PostgresSourceRepository(sql) }),
|
|
});
|
|
}
|
|
|
|
function freshIngestion(): {
|
|
repo: PostgresIngestionRepository;
|
|
runner: IngestionRunner;
|
|
lineage: LineageStore;
|
|
} {
|
|
const sql = postgres(TEST_DATABASE_URL as string, { prepare: false, ssl: false, max: 2 });
|
|
const repo = new PostgresIngestionRepository(sql);
|
|
const lineage = new LineageStore(repo);
|
|
return { repo, runner: new IngestionRunner(repo, new SchemaRegistry(repo), lineage), lineage };
|
|
}
|
|
|
|
run("P060 real Postgres — a restart preserves every source and every registry fact", () => {
|
|
it("is reachable (otherwise the rest of this suite proves nothing)", () => {
|
|
expect(reachable).toBe(true);
|
|
});
|
|
|
|
it("registers pending, refuses activation unclassified, then activates — all surviving a fresh process", async () => {
|
|
const writer = freshRegistry();
|
|
await writer.registerSource({
|
|
source_id: SOURCE,
|
|
tenant_id: TENANT,
|
|
project_id: "p_realinfra",
|
|
source_type: "file",
|
|
name: "Realinfra file drop",
|
|
owner_id: "usr_simon",
|
|
schedule: { mode: "batch", interval_seconds: 900, enabled: true },
|
|
});
|
|
|
|
// A DIFFERENT service instance over a DIFFERENT connection.
|
|
const reader = freshRegistry();
|
|
expect((await reader.sources.getSource(SOURCE)).status).toBe("pending");
|
|
await expect(reader.activateSource(SOURCE, TENANT)).rejects.toBeInstanceOf(SiaxError);
|
|
|
|
const classifier = freshRegistry();
|
|
await classifier.setRights({
|
|
source_id: SOURCE,
|
|
tenant_id: TENANT,
|
|
license_id: "proprietary:realinfra",
|
|
allowed_purposes: ["catalog"],
|
|
pii_allowed: false,
|
|
redistribution: "internal",
|
|
verified_at: "2026-09-04",
|
|
verified_by: "usr_simon",
|
|
});
|
|
await classifier.setProvenance({
|
|
source_id: SOURCE,
|
|
tenant_id: TENANT,
|
|
method: "bulk_download",
|
|
updated_at: new Date().toISOString(),
|
|
});
|
|
await classifier.setAuthorization({
|
|
source_id: SOURCE,
|
|
tenant_id: TENANT,
|
|
basis: "contract",
|
|
granted_by: "usr_simon",
|
|
granted_at: new Date().toISOString(),
|
|
});
|
|
|
|
const activator = freshRegistry();
|
|
expect((await activator.classifySource(SOURCE, TENANT)).classified).toBe(true);
|
|
expect((await activator.activateSource(SOURCE, TENANT, "usr_simon")).status).toBe("active");
|
|
|
|
// Read back through yet another fresh process AND directly in SQL, so the
|
|
// assertion cannot be satisfied by any in-process bookkeeping.
|
|
const verifier = freshRegistry();
|
|
const descriptor = await verifier.describeSource(SOURCE, TENANT);
|
|
expect(descriptor.source.status).toBe("active");
|
|
expect(descriptor.source.owner_id).toBe("usr_simon");
|
|
expect(descriptor.authorization?.basis).toBe("contract");
|
|
expect(descriptor.schedule?.interval_seconds).toBe(900);
|
|
expect(descriptor.classification?.classified).toBe(true);
|
|
|
|
const rows = await (adminSql as ReturnType<typeof postgres>)<
|
|
{ status: string; owner_id: string | null }[]
|
|
>`SELECT status, owner_id FROM opt_sources WHERE source_id = ${SOURCE}`;
|
|
expect(rows[0]).toEqual({ status: "active", owner_id: "usr_simon" });
|
|
});
|
|
|
|
it("a raw INSERT that omits status lands PENDING — the gate is in the schema too", async () => {
|
|
const sql = adminSql as ReturnType<typeof postgres>;
|
|
await sql`
|
|
INSERT INTO opt_sources (source_id, tenant_id, project_id, source_type, name)
|
|
VALUES ('src_realinfra_raw', ${TENANT}, 'p_realinfra', 'api', 'Raw insert')
|
|
ON CONFLICT (source_id) DO NOTHING`;
|
|
const rows = await sql<{ status: string }[]>`
|
|
SELECT status FROM opt_sources WHERE source_id = 'src_realinfra_raw'`;
|
|
expect(rows[0].status).toBe("pending");
|
|
});
|
|
});
|
|
|
|
run("P062/P063/P064/P067/P068 real Postgres — the durable ingestion plane", () => {
|
|
it("runs, crashes, and resumes across fresh processes with no gap and no duplicate", async () => {
|
|
const first = freshIngestion();
|
|
const delivered: string[] = [];
|
|
let pages = 0;
|
|
const input = {
|
|
source_id: SOURCE,
|
|
tenant_id: TENANT,
|
|
project_id: "p_realinfra",
|
|
mode: "batch" as const,
|
|
connector: new JsonlFileConnector(),
|
|
config: { file_path: filePath, partition_key: "companies", schema: SCHEMA, record_key_field: "org_number" },
|
|
page_size: 2,
|
|
output_ref: "0pt:dataset:ds_realinfra",
|
|
};
|
|
await expect(
|
|
first.runner.run({
|
|
...input,
|
|
sink: async (batch) => {
|
|
pages++;
|
|
if (pages === 2) throw new Error("simulated process death");
|
|
for (const e of batch) delivered.push(e.record_key);
|
|
},
|
|
}),
|
|
).rejects.toThrow(/simulated process death/);
|
|
expect(delivered).toHaveLength(2);
|
|
|
|
// Fresh everything — only Postgres carries the position forward.
|
|
const second = freshIngestion();
|
|
const report = await second.runner.run({
|
|
...input,
|
|
connector: new JsonlFileConnector(),
|
|
sink: async (batch) => {
|
|
for (const e of batch) delivered.push(e.record_key);
|
|
},
|
|
});
|
|
expect(report.status).toBe("succeeded");
|
|
expect(delivered).toHaveLength(ROWS.length);
|
|
expect(new Set(delivered).size).toBe(ROWS.length);
|
|
|
|
const cp = await second.repo.getCheckpoint(SOURCE, "companies");
|
|
expect(cp?.records_committed).toBe(ROWS.length);
|
|
});
|
|
|
|
it("the schema version, ledger, and lineage are all readable from a fresh process", async () => {
|
|
const fresh = freshIngestion();
|
|
expect((await new SchemaRegistry(fresh.repo).getLatest(SOURCE))?.schema_version).toBe(1);
|
|
expect(await fresh.repo.countLedger(SOURCE)).toBe(ROWS.length);
|
|
const upstream = await fresh.lineage.upstreamOf(TENANT, "0pt:dataset:ds_realinfra");
|
|
expect(upstream.length).toBeGreaterThanOrEqual(1);
|
|
expect(upstream[0].input_ref).toBe(`0pt:source:${SOURCE}`);
|
|
});
|
|
|
|
it("a repeated backfill is idempotent against the durable ledger", async () => {
|
|
const fresh = freshIngestion();
|
|
const emitted: string[] = [];
|
|
const report = await fresh.runner.backfill({
|
|
source_id: SOURCE,
|
|
tenant_id: TENANT,
|
|
project_id: "p_realinfra",
|
|
mode: "batch",
|
|
connector: new JsonlFileConnector(),
|
|
config: { file_path: filePath, partition_key: "companies", schema: SCHEMA, record_key_field: "org_number" },
|
|
sink: async (batch) => {
|
|
for (const e of batch) emitted.push(e.record_key);
|
|
},
|
|
page_size: 100,
|
|
range: { partition_key: "companies", from: "0", to: String(ROWS.length) },
|
|
});
|
|
expect(report.envelopes_emitted).toBe(0);
|
|
expect(report.duplicates_skipped).toBe(ROWS.length);
|
|
expect(emitted).toHaveLength(0);
|
|
});
|
|
|
|
it("quarantine survives to a fresh process with the full envelope", async () => {
|
|
const fresh = freshIngestion();
|
|
await fresh.runner.backfill({
|
|
source_id: SOURCE,
|
|
tenant_id: TENANT,
|
|
project_id: "p_realinfra",
|
|
mode: "batch",
|
|
connector: new JsonlFileConnector(),
|
|
config: { file_path: filePath, partition_key: "companies", schema: SCHEMA, record_key_field: "org_number" },
|
|
sink: async () => undefined,
|
|
page_size: 100,
|
|
quality_policy: {
|
|
source_id: SOURCE,
|
|
tenant_id: TENANT,
|
|
expectations: [
|
|
{ expectation_type: "expect_column_values_to_be_between", kwargs: { column: "employees", min_value: 0, max_value: 20 } },
|
|
],
|
|
on_violation: "quarantine",
|
|
},
|
|
range: { partition_key: "companies", from: "0", to: String(ROWS.length) },
|
|
});
|
|
const reader = freshIngestion();
|
|
const quarantined = await reader.repo.listQuarantine(SOURCE);
|
|
expect(quarantined.length).toBe(ROWS.length - 2);
|
|
expect(quarantined[0].envelope.record_key).toBeTruthy();
|
|
expect(quarantined[0].violations[0].expectation_type).toBe("expect_column_values_to_be_between");
|
|
});
|
|
});
|