166 lines
7.2 KiB
TypeScript
166 lines
7.2 KiB
TypeScript
/**
|
|
* P061 DoD: "two heterogeneous connectors pass the same conformance suite."
|
|
*
|
|
* This file is that proof. It runs the SAME function
|
|
* (src/connectors/conformance.ts::runConnectorConformance) against two
|
|
* connectors that share nothing but the contract:
|
|
*
|
|
* HttpJsonConnector — a real node:http server on a real socket, opaque
|
|
* cursor paging, network health check, timestamp
|
|
* backfill bounds.
|
|
* JsonlFileConnector — a real file on disk, line-offset cursor, filesystem
|
|
* health check, line-number backfill bounds, and CDC
|
|
* delete records the HTTP connector cannot produce.
|
|
*
|
|
* Neither connector is mocked. The HTTP fixture is a genuine server that
|
|
* implements paging server-side; the file fixture is a genuine file written
|
|
* to a temp dir. A connector that only "passed" against a stub would prove
|
|
* nothing about the contract.
|
|
*/
|
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
import http from "node:http";
|
|
import type { AddressInfo } from "node:net";
|
|
import { promises as fs } from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { runConnectorConformance, type ConformanceFixture } from "@/src/connectors/conformance";
|
|
import { HttpJsonConnector } from "@/src/connectors/http-json-connector";
|
|
import { JsonlFileConnector } from "@/src/connectors/jsonl-file-connector";
|
|
import type { RecordSchemaFields } from "@/src/schema/record-schema";
|
|
|
|
const SCHEMA: RecordSchemaFields = {
|
|
org_number: { type: "string", required: true },
|
|
name: { type: "string", required: true },
|
|
employees: { type: "number", required: false },
|
|
};
|
|
|
|
/** 5 records, shared by both fixtures so the suite's counts are comparable. */
|
|
const ROWS = [
|
|
{ org_number: "556001-0001", name: "Alfa AB", employees: 10, seq: 1 },
|
|
{ org_number: "556002-0002", name: "Beta AB", employees: 20, seq: 2 },
|
|
{ org_number: "556003-0003", name: "Gamma AB", employees: 30, seq: 3 },
|
|
{ org_number: "556004-0004", name: "Delta AB", employees: 40, seq: 4 },
|
|
{ org_number: "556005-0005", name: "Epsilon AB", employees: 50, seq: 5 },
|
|
];
|
|
|
|
// --- fixture 1: a real paged HTTP API ---------------------------------------
|
|
|
|
let server: http.Server;
|
|
let baseUrl = "";
|
|
|
|
beforeAll(async () => {
|
|
server = http.createServer((req, res) => {
|
|
const url = new URL(req.url ?? "/", "http://localhost");
|
|
if (url.pathname !== "/v1/records") {
|
|
res.writeHead(404).end("{}");
|
|
return;
|
|
}
|
|
const limit = Number(url.searchParams.get("limit") ?? "100");
|
|
const from = url.searchParams.get("from");
|
|
const to = url.searchParams.get("to");
|
|
if (from !== null && to !== null) {
|
|
// Backfill: bounds are `seq` values, from inclusive, to exclusive.
|
|
const slice = ROWS.filter((r) => r.seq >= Number(from) && r.seq < Number(to)).slice(0, limit);
|
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
res.end(JSON.stringify({ records: slice, next_cursor: null }));
|
|
return;
|
|
}
|
|
// Cursor is the `seq` of the last record already delivered.
|
|
const after = Number(url.searchParams.get("cursor") ?? "0");
|
|
const slice = ROWS.filter((r) => r.seq > after).slice(0, limit);
|
|
const next = slice.length > 0 ? String(slice[slice.length - 1].seq) : null;
|
|
const more = slice.length > 0 && slice[slice.length - 1].seq < ROWS[ROWS.length - 1].seq;
|
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
res.end(JSON.stringify({ records: slice, next_cursor: more ? next : null }));
|
|
});
|
|
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await new Promise<void>((resolve) => server.close(() => resolve()));
|
|
await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => undefined);
|
|
});
|
|
|
|
// --- fixture 2: a real JSONL file -------------------------------------------
|
|
|
|
const tmpDir = path.join(os.tmpdir(), `opt-conformance-${process.pid}`);
|
|
const jsonlPath = path.join(tmpDir, "companies.jsonl");
|
|
|
|
beforeAll(async () => {
|
|
await fs.mkdir(tmpDir, { recursive: true });
|
|
await fs.writeFile(
|
|
jsonlPath,
|
|
ROWS.map((r) => JSON.stringify({ org_number: r.org_number, name: r.name, employees: r.employees })).join("\n") + "\n",
|
|
"utf8",
|
|
);
|
|
});
|
|
|
|
const ctx = { tenant_id: "t_test", project_id: "p_test", source_id: "src_conf", run_id: "run_conf" };
|
|
|
|
describe("P061 connector contract — the same conformance suite, two heterogeneous connectors", () => {
|
|
it("HttpJsonConnector passes every check", async () => {
|
|
const fixture: ConformanceFixture = {
|
|
validConfig: {
|
|
base_url: baseUrl,
|
|
records_path: "/v1/records",
|
|
partition_key: "companies",
|
|
schema: SCHEMA,
|
|
record_key_field: "org_number",
|
|
cursor_field: "seq",
|
|
},
|
|
// Not an absolute URL and no schema — must be rejected without any IO.
|
|
invalidConfig: { base_url: "not-a-url", records_path: "v1/records", partition_key: "", record_key_field: "" },
|
|
ctx,
|
|
backfill: { from: "2", to: "4" },
|
|
expectedRecordCount: ROWS.length,
|
|
};
|
|
const report = await runConnectorConformance(new HttpJsonConnector(), fixture);
|
|
const failed = report.checks.filter((c) => !c.passed);
|
|
expect(failed.map((c) => `${c.name}: ${c.detail}`)).toEqual([]);
|
|
expect(report.passed).toBe(true);
|
|
expect(report.checks.length).toBeGreaterThanOrEqual(14);
|
|
});
|
|
|
|
it("JsonlFileConnector passes every check — the identical suite", async () => {
|
|
const fixture: ConformanceFixture = {
|
|
validConfig: {
|
|
file_path: jsonlPath,
|
|
partition_key: "companies",
|
|
schema: SCHEMA,
|
|
record_key_field: "org_number",
|
|
},
|
|
// Relative path, wrong extension, no schema.
|
|
invalidConfig: { file_path: "companies.csv", partition_key: "", record_key_field: "" },
|
|
ctx,
|
|
// File-connector bounds are LINE NUMBERS — deliberately a different
|
|
// vocabulary from the HTTP fixture's timestamps/sequence values, which
|
|
// is exactly the heterogeneity the shared contract has to survive.
|
|
backfill: { from: "1", to: "3" },
|
|
expectedRecordCount: ROWS.length,
|
|
};
|
|
const report = await runConnectorConformance(new JsonlFileConnector(), fixture);
|
|
const failed = report.checks.filter((c) => !c.passed);
|
|
expect(failed.map((c) => `${c.name}: ${c.detail}`)).toEqual([]);
|
|
expect(report.passed).toBe(true);
|
|
});
|
|
|
|
it("the suite actually FAILS a connector that breaks the contract (it is not decoration)", async () => {
|
|
// A connector that accepts any config at all — the classic fail-open bug.
|
|
class FailOpenConnector extends JsonlFileConnector {
|
|
override validate() {
|
|
return { valid: true, errors: [] };
|
|
}
|
|
}
|
|
const report = await runConnectorConformance(new FailOpenConnector(), {
|
|
validConfig: { file_path: jsonlPath, partition_key: "companies", schema: SCHEMA, record_key_field: "org_number" },
|
|
invalidConfig: { file_path: "nope.csv", partition_key: "", record_key_field: "" },
|
|
ctx,
|
|
backfill: { from: "1", to: "3" },
|
|
expectedRecordCount: ROWS.length,
|
|
});
|
|
expect(report.passed).toBe(false);
|
|
expect(report.checks.find((c) => c.name.startsWith("validate: rejects"))?.passed).toBe(false);
|
|
});
|
|
});
|