Files
0pt/tests/p0/ingestion-runner.test.ts

413 lines
17 KiB
TypeScript

/**
* The ingestion plane end to end:
*
* P062 — every connector's output becomes the same envelope, and a run that
* DIES mid-page resumes from the durable checkpoint with no gap and
* no double-delivery.
* P063 — a breaking schema change stops the run BEFORE the sink is called.
* P064 — quality failures block or quarantine according to the DECLARED
* policy, and a blocked page does not advance the checkpoint.
* P067 — a run's source -> output lineage is queryable afterwards.
* P068 — a repeated backfill is idempotent.
*
* The connectors here are the real ones (JsonlFileConnector against a real
* file on disk); the "crash" is a sink that throws, which is the closest
* faithful analogue of a process dying between the consumer write and the
* checkpoint commit.
*/
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
import { promises as fs } from "node:fs";
import os from "node:os";
import path from "node:path";
import { IngestionRunner, type IngestionRunInput } from "@/src/ingestion/ingestion-runner";
import { InMemoryIngestionRepository } from "@/src/repositories/in-memory-ingestion-repository";
import { SchemaRegistry } from "@/src/schema/schema-registry";
import { LineageStore } from "@/src/lineage/lineage-store";
import { JsonlFileConnector } from "@/src/connectors/jsonl-file-connector";
import { hashPayload } from "@/src/ingestion/envelope";
import { SiaxError } from "@/src/platform/errors";
import type { IngestionEnvelope } from "@/src/types/ingestion";
import type { RecordSchemaFields } from "@/src/schema/record-schema";
const SCHEMA: RecordSchemaFields = {
org_number: { type: "string", required: true },
name: { type: "string", required: true },
employees: { type: "number", required: false },
};
const tmpDir = path.join(os.tmpdir(), `opt-runner-${process.pid}`);
const filePath = path.join(tmpDir, "companies.jsonl");
const ROWS = Array.from({ length: 10 }, (_, i) => ({
org_number: `55600${i}-000${i}`,
name: `Bolag ${i} AB`,
employees: (i + 1) * 10,
}));
beforeAll(async () => {
await fs.mkdir(tmpDir, { recursive: true });
await fs.writeFile(filePath, ROWS.map((r) => JSON.stringify(r)).join("\n") + "\n", "utf8");
});
afterAll(async () => {
await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => undefined);
});
let repo: InMemoryIngestionRepository;
let runner: IngestionRunner;
let lineage: LineageStore;
beforeEach(() => {
repo = new InMemoryIngestionRepository();
lineage = new LineageStore(repo);
runner = new IngestionRunner(repo, new SchemaRegistry(repo), lineage);
});
function baseInput(sink: IngestionRunInput["sink"], overrides: Partial<IngestionRunInput> = {}): IngestionRunInput {
return {
source_id: "src_ing",
tenant_id: "t_test",
project_id: "p_test",
mode: "batch",
connector: new JsonlFileConnector(),
config: { file_path: filePath, partition_key: "companies", schema: SCHEMA, record_key_field: "org_number" },
sink,
page_size: 3,
output_ref: "0pt:dataset:ds_companies",
...overrides,
};
}
describe("P062 common envelope", () => {
it("normalizes every record into one envelope shape with provenance and a monotonic sequence", async () => {
const seen: IngestionEnvelope[] = [];
const report = await runner.run(baseInput(async (batch) => void seen.push(...batch)));
expect(report.status).toBe("succeeded");
expect(seen).toHaveLength(ROWS.length);
expect(seen.map((e) => e.sequence)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
for (const envelope of seen) {
expect(envelope.source_id).toBe("src_ing");
expect(envelope.tenant_id).toBe("t_test");
expect(envelope.mode).toBe("batch");
expect(envelope.operation).toBe("upsert");
expect(envelope.partition_key).toBe("companies");
expect(envelope.schema_version).toBe(1);
expect(envelope.provenance.connector_id).toBe("jsonl_file");
expect(envelope.provenance.connector_version).toBe("jsonl-file-connector@1");
expect(envelope.provenance.run_id).toBe(report.run_id);
expect(envelope.ingested_at).toMatch(/^\d{4}-\d{2}-\d{2}T/);
}
});
it("the durable checkpoint advances to the end of the partition", async () => {
await runner.run(baseInput(async () => undefined));
const checkpoint = await repo.getCheckpoint("src_ing", "companies");
expect(checkpoint?.cursor).toBe(String(ROWS.length));
expect(checkpoint?.sequence).toBe(ROWS.length);
expect(checkpoint?.records_committed).toBe(ROWS.length);
});
it("a run that dies inside the sink resumes with NO gap and NO double delivery", async () => {
const delivered: string[] = [];
let pages = 0;
// First run: the sink dies on the third page (records 7-9), AFTER two
// pages have been committed. This is the crash window the ordering exists
// to protect: the checkpoint for page 3 was never committed.
const crashing = baseInput(async (batch) => {
pages++;
if (pages === 3) throw new Error("process died mid-write");
for (const e of batch) delivered.push(e.record_key);
});
await expect(runner.run(crashing)).rejects.toThrow(/process died mid-write/);
expect(delivered).toHaveLength(6);
const afterCrash = await repo.getCheckpoint("src_ing", "companies");
expect(afterCrash?.cursor).toBe("6");
// Second run: a FRESH runner and a FRESH connector, exactly as a restarted
// process would be — only the durable checkpoint carries over.
const resumed = new IngestionRunner(repo, new SchemaRegistry(repo), lineage);
const report = await resumed.run(
baseInput(async (batch) => {
for (const e of batch) delivered.push(e.record_key);
}),
);
expect(report.status).toBe("succeeded");
// Every record exactly once across both runs: no gap, no duplicate.
expect(delivered).toHaveLength(ROWS.length);
expect(new Set(delivered).size).toBe(ROWS.length);
expect([...delivered].sort()).toEqual(ROWS.map((r) => r.org_number).sort());
});
it("refuses a mode the connector does not serve", async () => {
// JsonlFileConnector declares batch + cdc; it never claims "stream".
await expect(runner.run(baseInput(async () => undefined, { mode: "stream" }))).rejects.toThrow(
/does not serve mode "stream"/,
);
});
});
describe("P063 breaking schema is caught before the consumer sees anything", () => {
it("fails the run BEFORE the sink is called when the shape breaks the declared policy", async () => {
// v1 registered by a clean run.
await runner.run(baseInput(async () => undefined));
expect((await new SchemaRegistry(repo).getLatest("src_ing"))?.schema_version).toBe(1);
// The source now claims a new REQUIRED field — backward-breaking.
let sinkCalls = 0;
const breaking = baseInput(async () => void sinkCalls++, {
source_id: "src_ing",
config: {
file_path: filePath,
partition_key: "companies",
schema: { ...SCHEMA, sni_code: { type: "string", required: true } },
record_key_field: "org_number",
},
});
const err = await runner.run(breaking).catch((e: unknown) => e);
expect(err).toBeInstanceOf(SiaxError);
expect((err as SiaxError).code).toBe("CONFLICT");
expect((err as SiaxError).message).toContain("refusing to ingest");
// The whole point: no consumer write happened.
expect(sinkCalls).toBe(0);
});
it("catches a record that stopped conforming mid-stream, before emitting the page", async () => {
// A file whose DECLARED schema is unchanged but whose DATA drifted:
// `employees` arrives as a string. A distinct source_id so this run starts
// from a clean checkpoint rather than resuming past the file.
const driftPath = path.join(tmpDir, "drift.jsonl");
await fs.writeFile(
driftPath,
JSON.stringify({ org_number: "556999-9999", name: "Drift AB", employees: "many" }) + "\n",
"utf8",
);
let sinkCalls = 0;
const drifting = baseInput(async () => void sinkCalls++, {
source_id: "src_drift",
config: { file_path: driftPath, partition_key: "companies", schema: SCHEMA, record_key_field: "org_number" },
});
await expect(runner.run(drifting)).rejects.toThrow(/does not conform to registered schema v1/);
expect(sinkCalls).toBe(0);
});
});
describe("P064 quality enforcement follows the DECLARED policy", () => {
const blockingExpectation = {
expectation_type: "expect_column_values_to_be_between",
kwargs: { column: "employees", min_value: 0, max_value: 50 },
};
it("'block' stops the run and does NOT advance the checkpoint past the bad page", async () => {
const seen: IngestionEnvelope[] = [];
const report = await runner.run(
baseInput(async (b) => void seen.push(...b), {
quality_policy: {
source_id: "src_ing",
tenant_id: "t_test",
expectations: [blockingExpectation],
on_violation: "block",
},
}),
);
expect(report.quality_policy).toBe("block");
const partition = report.partitions[0];
expect(partition.stopped_because).toBe("blocked_by_quality");
// Records 0-4 have employees <= 50; the block fires on the page containing
// employees 60, i.e. page 3 (records 6,7,8 -> 70,80,90 ... employees 60 is
// record index 5 on page 2). Whatever the exact page, nothing from the
// blocked page was emitted and the checkpoint stayed behind it.
expect(partition.envelopes_emitted).toBeLessThan(ROWS.length);
const checkpoint = await repo.getCheckpoint("src_ing", "companies");
expect(Number(checkpoint?.cursor ?? "0")).toBeLessThan(ROWS.length);
expect(seen.every((e) => Number(e.payload?.employees) <= 50)).toBe(true);
});
it("'quarantine' persists the failing records, lets the clean ones through, and advances", async () => {
const seen: IngestionEnvelope[] = [];
const report = await runner.run(
baseInput(async (b) => void seen.push(...b), {
quality_policy: {
source_id: "src_ing",
tenant_id: "t_test",
expectations: [blockingExpectation],
on_violation: "quarantine",
},
}),
);
expect(report.quality_policy).toBe("quarantine");
const partition = report.partitions[0];
expect(partition.stopped_because).toBeUndefined();
expect(partition.quarantined).toBe(5); // employees 60..100
expect(partition.envelopes_emitted).toBe(5); // employees 10..50
expect(seen.every((e) => Number(e.payload?.employees) <= 50)).toBe(true);
const quarantined = await repo.listQuarantine("src_ing");
expect(quarantined).toHaveLength(5);
expect(quarantined[0].violations[0].expectation_type).toBe("expect_column_values_to_be_between");
// The full envelope is preserved — a quarantined record is recoverable,
// not just counted.
expect(quarantined[0].envelope.payload).toBeTruthy();
const checkpoint = await repo.getCheckpoint("src_ing", "companies");
expect(checkpoint?.cursor).toBe(String(ROWS.length));
});
it("'warn' lets everything through but still reports the violations", async () => {
const seen: IngestionEnvelope[] = [];
const report = await runner.run(
baseInput(async (b) => void seen.push(...b), {
quality_policy: {
source_id: "src_ing",
tenant_id: "t_test",
expectations: [blockingExpectation],
on_violation: "warn",
},
}),
);
expect(seen).toHaveLength(ROWS.length);
expect(report.partitions[0].quality_violations).toBe(5);
expect(report.partitions[0].quarantined).toBe(0);
});
it("no declared policy is reported as 'none_declared', never as a pass", async () => {
const report = await runner.run(baseInput(async () => undefined));
expect(report.quality_policy).toBe("none_declared");
});
it("an expectation the local engine cannot execute is a violation, not a silent pass", async () => {
const report = await runner.run(
baseInput(async () => undefined, {
quality_policy: {
source_id: "src_ing",
tenant_id: "t_test",
expectations: [{ expectation_type: "expect_column_kl_divergence_to_be_less_than", kwargs: { column: "name" } }],
on_violation: "quarantine",
},
}),
);
expect(report.partitions[0].quarantined).toBe(ROWS.length);
const quarantined = await repo.listQuarantine("src_ing");
expect(quarantined[0].violations[0].detail).toContain("not executable by the local engine");
});
});
describe("P067 lineage is queryable after the run", () => {
it("answers 'which source fed this output' and 'what did this source produce'", async () => {
const report = await runner.run(baseInput(async () => undefined, { correlation_id: "corr-lineage-1" }));
expect(report.lineage_edges).toBe(1);
const upstream = await lineage.upstreamOf("t_test", "0pt:dataset:ds_companies");
expect(upstream).toHaveLength(1);
expect(upstream[0].input_name).toBe("source/src_ing");
expect(upstream[0].input_ref).toBe("0pt:source:src_ing");
expect(upstream[0].capability_id).toBe("0pt.ingestion.run.v1");
expect(upstream[0].run_id).toBe(report.run_id);
expect(upstream[0].correlation_id).toBe("corr-lineage-1");
const downstream = await lineage.downstreamOf("t_test", "source/src_ing");
expect(downstream.map((e) => e.output_name)).toEqual(["0pt:dataset:ds_companies"]);
// Tenant-scoped: another tenant sees nothing.
expect(await lineage.upstreamOf("t_other", "0pt:dataset:ds_companies")).toEqual([]);
});
it("re-emitting the same run id does not duplicate edges (deterministic edge ids)", async () => {
await runner.run(baseInput(async () => undefined, { run_id: "run_fixed" }));
// A second identical run of the same id — e.g. an idempotent retry.
const repo2Runner = new IngestionRunner(repo, new SchemaRegistry(repo), lineage);
await repo2Runner.run(baseInput(async () => undefined, { run_id: "run_fixed" }));
const edges = await lineage.upstreamOf("t_test", "0pt:dataset:ds_companies");
expect(edges).toHaveLength(1);
});
});
describe("P068 backfill is bounded and idempotent", () => {
it("a repeated identical backfill emits nothing the second time", async () => {
const first: IngestionEnvelope[] = [];
const firstReport = await runner.backfill({
...baseInput(async (b) => void first.push(...b)),
range: { partition_key: "companies", from: "0", to: "5" },
page_size: 100,
});
expect(firstReport.envelopes_emitted).toBe(5);
expect(firstReport.duplicates_skipped).toBe(0);
expect(first).toHaveLength(5);
const second: IngestionEnvelope[] = [];
const secondReport = await runner.backfill({
...baseInput(async (b) => void second.push(...b)),
range: { partition_key: "companies", from: "0", to: "5" },
page_size: 100,
});
expect(secondReport.envelopes_emitted).toBe(0);
expect(secondReport.duplicates_skipped).toBe(5);
expect(second).toHaveLength(0);
});
it("a backfill does NOT move the live checkpoint", async () => {
await runner.run(baseInput(async () => undefined));
const before = await repo.getCheckpoint("src_ing", "companies");
await runner.backfill({
...baseInput(async () => undefined),
range: { partition_key: "companies", from: "0", to: "3" },
});
const after = await repo.getCheckpoint("src_ing", "companies");
expect(after).toEqual(before);
});
it("a backfill overlapping a completed run re-delivers nothing (the ledger already has it)", async () => {
await runner.run(baseInput(async () => undefined));
const emitted: IngestionEnvelope[] = [];
const report = await runner.backfill({
...baseInput(async (b) => void emitted.push(...b)),
range: { partition_key: "companies", from: "0", to: "10" },
page_size: 100,
});
expect(report.duplicates_skipped).toBe(10);
expect(emitted).toHaveLength(0);
});
it("a record whose CONTENT changed is re-emitted — idempotency is not amnesia", async () => {
await runner.run(baseInput(async () => undefined));
const changedPath = path.join(tmpDir, "changed.jsonl");
await fs.writeFile(
changedPath,
JSON.stringify({ ...ROWS[0], name: "Bolag 0 AB (namnändrad)" }) + "\n",
"utf8",
);
const emitted: IngestionEnvelope[] = [];
const report = await runner.backfill({
...baseInput(async (b) => void emitted.push(...b), {
config: { file_path: changedPath, partition_key: "companies", schema: SCHEMA, record_key_field: "org_number" },
}),
range: { partition_key: "companies", from: "0", to: "1" },
page_size: 100,
});
expect(report.changed).toBe(1);
expect(report.envelopes_emitted).toBe(1);
expect(emitted[0].payload?.name).toBe("Bolag 0 AB (namnändrad)");
});
it("an unbounded backfill is refused by the connector contract", async () => {
await expect(
runner.backfill({
...baseInput(async () => undefined),
range: { partition_key: "companies", from: "0", to: "" },
}),
).rejects.toThrow(/unbounded backfill|requires both/);
});
});
describe("envelope hashing", () => {
it("is stable across key order and distinguishes a real content change", () => {
expect(hashPayload({ a: 1, b: 2 })).toBe(hashPayload({ b: 2, a: 1 }));
expect(hashPayload({ a: 1, b: 2 })).not.toBe(hashPayload({ a: 1, b: 3 }));
// null and absent are different data, and must hash differently.
expect(hashPayload({ a: 1, b: null })).not.toBe(hashPayload({ a: 1 }));
expect(hashPayload(null)).toBe("sha256:deleted");
});
});