/** * The ingestion plane end to end: * * P062 — every connector's output becomes the same envelope, and a run that * DIES mid-page resumes from the durable checkpoint with no gap and * no double-delivery. * P063 — a breaking schema change stops the run BEFORE the sink is called. * P064 — quality failures block or quarantine according to the DECLARED * policy, and a blocked page does not advance the checkpoint. * P067 — a run's source -> output lineage is queryable afterwards. * P068 — a repeated backfill is idempotent. * * The connectors here are the real ones (JsonlFileConnector against a real * file on disk); the "crash" is a sink that throws, which is the closest * faithful analogue of a process dying between the consumer write and the * checkpoint commit. */ import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { promises as fs } from "node:fs"; import os from "node:os"; import path from "node:path"; import { IngestionRunner, type IngestionRunInput } from "@/src/ingestion/ingestion-runner"; import { InMemoryIngestionRepository } from "@/src/repositories/in-memory-ingestion-repository"; import { SchemaRegistry } from "@/src/schema/schema-registry"; import { LineageStore } from "@/src/lineage/lineage-store"; import { JsonlFileConnector } from "@/src/connectors/jsonl-file-connector"; import { hashPayload } from "@/src/ingestion/envelope"; import { SiaxError } from "@/src/platform/errors"; import type { IngestionEnvelope } from "@/src/types/ingestion"; import type { RecordSchemaFields } from "@/src/schema/record-schema"; const SCHEMA: RecordSchemaFields = { org_number: { type: "string", required: true }, name: { type: "string", required: true }, employees: { type: "number", required: false }, }; const tmpDir = path.join(os.tmpdir(), `opt-runner-${process.pid}`); const filePath = path.join(tmpDir, "companies.jsonl"); const ROWS = Array.from({ length: 10 }, (_, i) => ({ org_number: `55600${i}-000${i}`, name: `Bolag ${i} AB`, employees: (i + 1) * 10, })); beforeAll(async () => { await fs.mkdir(tmpDir, { recursive: true }); await fs.writeFile(filePath, ROWS.map((r) => JSON.stringify(r)).join("\n") + "\n", "utf8"); }); afterAll(async () => { await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => undefined); }); let repo: InMemoryIngestionRepository; let runner: IngestionRunner; let lineage: LineageStore; beforeEach(() => { repo = new InMemoryIngestionRepository(); lineage = new LineageStore(repo); runner = new IngestionRunner(repo, new SchemaRegistry(repo), lineage); }); function baseInput(sink: IngestionRunInput["sink"], overrides: Partial = {}): IngestionRunInput { return { source_id: "src_ing", tenant_id: "t_test", project_id: "p_test", mode: "batch", connector: new JsonlFileConnector(), config: { file_path: filePath, partition_key: "companies", schema: SCHEMA, record_key_field: "org_number" }, sink, page_size: 3, output_ref: "0pt:dataset:ds_companies", ...overrides, }; } describe("P062 common envelope", () => { it("normalizes every record into one envelope shape with provenance and a monotonic sequence", async () => { const seen: IngestionEnvelope[] = []; const report = await runner.run(baseInput(async (batch) => void seen.push(...batch))); expect(report.status).toBe("succeeded"); expect(seen).toHaveLength(ROWS.length); expect(seen.map((e) => e.sequence)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); for (const envelope of seen) { expect(envelope.source_id).toBe("src_ing"); expect(envelope.tenant_id).toBe("t_test"); expect(envelope.mode).toBe("batch"); expect(envelope.operation).toBe("upsert"); expect(envelope.partition_key).toBe("companies"); expect(envelope.schema_version).toBe(1); expect(envelope.provenance.connector_id).toBe("jsonl_file"); expect(envelope.provenance.connector_version).toBe("jsonl-file-connector@1"); expect(envelope.provenance.run_id).toBe(report.run_id); expect(envelope.ingested_at).toMatch(/^\d{4}-\d{2}-\d{2}T/); } }); it("the durable checkpoint advances to the end of the partition", async () => { await runner.run(baseInput(async () => undefined)); const checkpoint = await repo.getCheckpoint("src_ing", "companies"); expect(checkpoint?.cursor).toBe(String(ROWS.length)); expect(checkpoint?.sequence).toBe(ROWS.length); expect(checkpoint?.records_committed).toBe(ROWS.length); }); it("a run that dies inside the sink resumes with NO gap and NO double delivery", async () => { const delivered: string[] = []; let pages = 0; // First run: the sink dies on the third page (records 7-9), AFTER two // pages have been committed. This is the crash window the ordering exists // to protect: the checkpoint for page 3 was never committed. const crashing = baseInput(async (batch) => { pages++; if (pages === 3) throw new Error("process died mid-write"); for (const e of batch) delivered.push(e.record_key); }); await expect(runner.run(crashing)).rejects.toThrow(/process died mid-write/); expect(delivered).toHaveLength(6); const afterCrash = await repo.getCheckpoint("src_ing", "companies"); expect(afterCrash?.cursor).toBe("6"); // Second run: a FRESH runner and a FRESH connector, exactly as a restarted // process would be — only the durable checkpoint carries over. const resumed = new IngestionRunner(repo, new SchemaRegistry(repo), lineage); const report = await resumed.run( baseInput(async (batch) => { for (const e of batch) delivered.push(e.record_key); }), ); expect(report.status).toBe("succeeded"); // Every record exactly once across both runs: no gap, no duplicate. expect(delivered).toHaveLength(ROWS.length); expect(new Set(delivered).size).toBe(ROWS.length); expect([...delivered].sort()).toEqual(ROWS.map((r) => r.org_number).sort()); }); it("refuses a mode the connector does not serve", async () => { // JsonlFileConnector declares batch + cdc; it never claims "stream". await expect(runner.run(baseInput(async () => undefined, { mode: "stream" }))).rejects.toThrow( /does not serve mode "stream"/, ); }); }); describe("P063 breaking schema is caught before the consumer sees anything", () => { it("fails the run BEFORE the sink is called when the shape breaks the declared policy", async () => { // v1 registered by a clean run. await runner.run(baseInput(async () => undefined)); expect((await new SchemaRegistry(repo).getLatest("src_ing"))?.schema_version).toBe(1); // The source now claims a new REQUIRED field — backward-breaking. let sinkCalls = 0; const breaking = baseInput(async () => void sinkCalls++, { source_id: "src_ing", config: { file_path: filePath, partition_key: "companies", schema: { ...SCHEMA, sni_code: { type: "string", required: true } }, record_key_field: "org_number", }, }); const err = await runner.run(breaking).catch((e: unknown) => e); expect(err).toBeInstanceOf(SiaxError); expect((err as SiaxError).code).toBe("CONFLICT"); expect((err as SiaxError).message).toContain("refusing to ingest"); // The whole point: no consumer write happened. expect(sinkCalls).toBe(0); }); it("catches a record that stopped conforming mid-stream, before emitting the page", async () => { // A file whose DECLARED schema is unchanged but whose DATA drifted: // `employees` arrives as a string. A distinct source_id so this run starts // from a clean checkpoint rather than resuming past the file. const driftPath = path.join(tmpDir, "drift.jsonl"); await fs.writeFile( driftPath, JSON.stringify({ org_number: "556999-9999", name: "Drift AB", employees: "many" }) + "\n", "utf8", ); let sinkCalls = 0; const drifting = baseInput(async () => void sinkCalls++, { source_id: "src_drift", config: { file_path: driftPath, partition_key: "companies", schema: SCHEMA, record_key_field: "org_number" }, }); await expect(runner.run(drifting)).rejects.toThrow(/does not conform to registered schema v1/); expect(sinkCalls).toBe(0); }); }); describe("P064 quality enforcement follows the DECLARED policy", () => { const blockingExpectation = { expectation_type: "expect_column_values_to_be_between", kwargs: { column: "employees", min_value: 0, max_value: 50 }, }; it("'block' stops the run and does NOT advance the checkpoint past the bad page", async () => { const seen: IngestionEnvelope[] = []; const report = await runner.run( baseInput(async (b) => void seen.push(...b), { quality_policy: { source_id: "src_ing", tenant_id: "t_test", expectations: [blockingExpectation], on_violation: "block", }, }), ); expect(report.quality_policy).toBe("block"); const partition = report.partitions[0]; expect(partition.stopped_because).toBe("blocked_by_quality"); // Records 0-4 have employees <= 50; the block fires on the page containing // employees 60, i.e. page 3 (records 6,7,8 -> 70,80,90 ... employees 60 is // record index 5 on page 2). Whatever the exact page, nothing from the // blocked page was emitted and the checkpoint stayed behind it. expect(partition.envelopes_emitted).toBeLessThan(ROWS.length); const checkpoint = await repo.getCheckpoint("src_ing", "companies"); expect(Number(checkpoint?.cursor ?? "0")).toBeLessThan(ROWS.length); expect(seen.every((e) => Number(e.payload?.employees) <= 50)).toBe(true); }); it("'quarantine' persists the failing records, lets the clean ones through, and advances", async () => { const seen: IngestionEnvelope[] = []; const report = await runner.run( baseInput(async (b) => void seen.push(...b), { quality_policy: { source_id: "src_ing", tenant_id: "t_test", expectations: [blockingExpectation], on_violation: "quarantine", }, }), ); expect(report.quality_policy).toBe("quarantine"); const partition = report.partitions[0]; expect(partition.stopped_because).toBeUndefined(); expect(partition.quarantined).toBe(5); // employees 60..100 expect(partition.envelopes_emitted).toBe(5); // employees 10..50 expect(seen.every((e) => Number(e.payload?.employees) <= 50)).toBe(true); const quarantined = await repo.listQuarantine("src_ing"); expect(quarantined).toHaveLength(5); expect(quarantined[0].violations[0].expectation_type).toBe("expect_column_values_to_be_between"); // The full envelope is preserved — a quarantined record is recoverable, // not just counted. expect(quarantined[0].envelope.payload).toBeTruthy(); const checkpoint = await repo.getCheckpoint("src_ing", "companies"); expect(checkpoint?.cursor).toBe(String(ROWS.length)); }); it("'warn' lets everything through but still reports the violations", async () => { const seen: IngestionEnvelope[] = []; const report = await runner.run( baseInput(async (b) => void seen.push(...b), { quality_policy: { source_id: "src_ing", tenant_id: "t_test", expectations: [blockingExpectation], on_violation: "warn", }, }), ); expect(seen).toHaveLength(ROWS.length); expect(report.partitions[0].quality_violations).toBe(5); expect(report.partitions[0].quarantined).toBe(0); }); it("no declared policy is reported as 'none_declared', never as a pass", async () => { const report = await runner.run(baseInput(async () => undefined)); expect(report.quality_policy).toBe("none_declared"); }); it("an expectation the local engine cannot execute is a violation, not a silent pass", async () => { const report = await runner.run( baseInput(async () => undefined, { quality_policy: { source_id: "src_ing", tenant_id: "t_test", expectations: [{ expectation_type: "expect_column_kl_divergence_to_be_less_than", kwargs: { column: "name" } }], on_violation: "quarantine", }, }), ); expect(report.partitions[0].quarantined).toBe(ROWS.length); const quarantined = await repo.listQuarantine("src_ing"); expect(quarantined[0].violations[0].detail).toContain("not executable by the local engine"); }); }); describe("P067 lineage is queryable after the run", () => { it("answers 'which source fed this output' and 'what did this source produce'", async () => { const report = await runner.run(baseInput(async () => undefined, { correlation_id: "corr-lineage-1" })); expect(report.lineage_edges).toBe(1); const upstream = await lineage.upstreamOf("t_test", "0pt:dataset:ds_companies"); expect(upstream).toHaveLength(1); expect(upstream[0].input_name).toBe("source/src_ing"); expect(upstream[0].input_ref).toBe("0pt:source:src_ing"); expect(upstream[0].capability_id).toBe("0pt.ingestion.run.v1"); expect(upstream[0].run_id).toBe(report.run_id); expect(upstream[0].correlation_id).toBe("corr-lineage-1"); const downstream = await lineage.downstreamOf("t_test", "source/src_ing"); expect(downstream.map((e) => e.output_name)).toEqual(["0pt:dataset:ds_companies"]); // Tenant-scoped: another tenant sees nothing. expect(await lineage.upstreamOf("t_other", "0pt:dataset:ds_companies")).toEqual([]); }); it("re-emitting the same run id does not duplicate edges (deterministic edge ids)", async () => { await runner.run(baseInput(async () => undefined, { run_id: "run_fixed" })); // A second identical run of the same id — e.g. an idempotent retry. const repo2Runner = new IngestionRunner(repo, new SchemaRegistry(repo), lineage); await repo2Runner.run(baseInput(async () => undefined, { run_id: "run_fixed" })); const edges = await lineage.upstreamOf("t_test", "0pt:dataset:ds_companies"); expect(edges).toHaveLength(1); }); }); describe("P068 backfill is bounded and idempotent", () => { it("a repeated identical backfill emits nothing the second time", async () => { const first: IngestionEnvelope[] = []; const firstReport = await runner.backfill({ ...baseInput(async (b) => void first.push(...b)), range: { partition_key: "companies", from: "0", to: "5" }, page_size: 100, }); expect(firstReport.envelopes_emitted).toBe(5); expect(firstReport.duplicates_skipped).toBe(0); expect(first).toHaveLength(5); const second: IngestionEnvelope[] = []; const secondReport = await runner.backfill({ ...baseInput(async (b) => void second.push(...b)), range: { partition_key: "companies", from: "0", to: "5" }, page_size: 100, }); expect(secondReport.envelopes_emitted).toBe(0); expect(secondReport.duplicates_skipped).toBe(5); expect(second).toHaveLength(0); }); it("a backfill does NOT move the live checkpoint", async () => { await runner.run(baseInput(async () => undefined)); const before = await repo.getCheckpoint("src_ing", "companies"); await runner.backfill({ ...baseInput(async () => undefined), range: { partition_key: "companies", from: "0", to: "3" }, }); const after = await repo.getCheckpoint("src_ing", "companies"); expect(after).toEqual(before); }); it("a backfill overlapping a completed run re-delivers nothing (the ledger already has it)", async () => { await runner.run(baseInput(async () => undefined)); const emitted: IngestionEnvelope[] = []; const report = await runner.backfill({ ...baseInput(async (b) => void emitted.push(...b)), range: { partition_key: "companies", from: "0", to: "10" }, page_size: 100, }); expect(report.duplicates_skipped).toBe(10); expect(emitted).toHaveLength(0); }); it("a record whose CONTENT changed is re-emitted — idempotency is not amnesia", async () => { await runner.run(baseInput(async () => undefined)); const changedPath = path.join(tmpDir, "changed.jsonl"); await fs.writeFile( changedPath, JSON.stringify({ ...ROWS[0], name: "Bolag 0 AB (namnändrad)" }) + "\n", "utf8", ); const emitted: IngestionEnvelope[] = []; const report = await runner.backfill({ ...baseInput(async (b) => void emitted.push(...b), { config: { file_path: changedPath, partition_key: "companies", schema: SCHEMA, record_key_field: "org_number" }, }), range: { partition_key: "companies", from: "0", to: "1" }, page_size: 100, }); expect(report.changed).toBe(1); expect(report.envelopes_emitted).toBe(1); expect(emitted[0].payload?.name).toBe("Bolag 0 AB (namnändrad)"); }); it("an unbounded backfill is refused by the connector contract", async () => { await expect( runner.backfill({ ...baseInput(async () => undefined), range: { partition_key: "companies", from: "0", to: "" }, }), ).rejects.toThrow(/unbounded backfill|requires both/); }); }); describe("envelope hashing", () => { it("is stable across key order and distinguishes a real content change", () => { expect(hashPayload({ a: 1, b: 2 })).toBe(hashPayload({ b: 2, a: 1 })); expect(hashPayload({ a: 1, b: 2 })).not.toBe(hashPayload({ a: 1, b: 3 })); // null and absent are different data, and must hash differently. expect(hashPayload({ a: 1, b: null })).not.toBe(hashPayload({ a: 1 })); expect(hashPayload(null)).toBe("sha256:deleted"); }); });