195 lines
9.8 KiB
TypeScript
195 lines
9.8 KiB
TypeScript
/**
|
|
* Connector-level proof for postgres-watermark-connector.ts. No real
|
|
* Postgres: `queryImpl` is a fixture that runs the SAME SQL text the real
|
|
* connector builds against an in-memory row set, so the watermark math and
|
|
* the tie-break are proven precisely rather than just "some rows came back".
|
|
*/
|
|
import { describe, expect, it } from "vitest";
|
|
import {
|
|
PostgresWatermarkConnector,
|
|
checksumRows,
|
|
decodeCursor,
|
|
encodeCursor,
|
|
type QueryImpl,
|
|
type SourceRow,
|
|
} from "@/src/connectors/postgres-watermark-connector";
|
|
import type { RecordSchemaFields } from "@/src/schema/record-schema";
|
|
|
|
const SCHEMA: RecordSchemaFields = {
|
|
id: { type: "string", required: true },
|
|
vehicle_id: { type: "string", required: false },
|
|
updated_at: { type: "string", required: true },
|
|
};
|
|
|
|
const CONFIG = {
|
|
table: "app.dealer_stock_entry",
|
|
partition_key: "dealer_stock_entry",
|
|
columns: ["id", "vehicle_id", "updated_at"],
|
|
record_key_column: "id",
|
|
watermark_column: "updated_at",
|
|
schema: SCHEMA,
|
|
};
|
|
|
|
/** A fixture query engine that actually parses/applies the connector's WHERE/ORDER BY/LIMIT against an in-memory table — not a stub that ignores its arguments. */
|
|
function fixtureQuery(rows: SourceRow[]): { query: QueryImpl; calls: Array<{ sql: string; params: unknown[] }> } {
|
|
const calls: Array<{ sql: string; params: unknown[] }> = [];
|
|
const query: QueryImpl = async (sql, params) => {
|
|
calls.push({ sql, params });
|
|
let filtered = rows;
|
|
if (sql.includes("WHERE")) {
|
|
const [watermark, id] = params as [string, string, number];
|
|
filtered = rows.filter(
|
|
(r) => (r.updated_at as string) > watermark || ((r.updated_at as string) === watermark && (r.id as string) > id),
|
|
);
|
|
}
|
|
const sorted = [...filtered].sort((a, b) => {
|
|
const w = (a.updated_at as string).localeCompare(b.updated_at as string);
|
|
return w !== 0 ? w : (a.id as string).localeCompare(b.id as string);
|
|
});
|
|
const limit = Number(params[params.length - 1]);
|
|
return sorted.slice(0, limit);
|
|
};
|
|
return { query, calls };
|
|
}
|
|
|
|
const ROWS: SourceRow[] = [
|
|
{ id: "a1", vehicle_id: "v1", updated_at: "2026-09-01T00:00:00.000Z" },
|
|
{ id: "a2", vehicle_id: "v2", updated_at: "2026-09-01T00:00:00.000Z" }, // same updated_at as a1 — the tie-break case
|
|
{ id: "a3", vehicle_id: "v1", updated_at: "2026-09-02T00:00:00.000Z" },
|
|
{ id: "a4", vehicle_id: "v3", updated_at: "2026-09-03T00:00:00.000Z" },
|
|
];
|
|
|
|
describe("PostgresWatermarkConnector — validate()", () => {
|
|
it("rejects a config pointed at the 98M-row vehicle table", () => {
|
|
const connector = new PostgresWatermarkConnector(fixtureQuery([]).query);
|
|
const verdict = connector.validate({ ...CONFIG, table: "vehiq.vehicle" });
|
|
expect(verdict.valid).toBe(false);
|
|
expect(verdict.errors.join(" ")).toMatch(/vehicle/i);
|
|
});
|
|
|
|
it("rejects SELECT * (an empty/missing columns list, or a literal '*')", () => {
|
|
const connector = new PostgresWatermarkConnector(fixtureQuery([]).query);
|
|
expect(connector.validate({ ...CONFIG, columns: [] }).valid).toBe(false);
|
|
expect(connector.validate({ ...CONFIG, columns: ["*"] }).valid).toBe(false);
|
|
});
|
|
|
|
it("accepts a well-formed config", () => {
|
|
const connector = new PostgresWatermarkConnector(fixtureQuery([]).query);
|
|
expect(connector.validate(CONFIG).valid).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe("PostgresWatermarkConnector — watermark + limit correctness", () => {
|
|
it("first read (no cursor) honours LIMIT and orders by (updated_at, id)", async () => {
|
|
const { query } = fixtureQuery(ROWS);
|
|
const connector = new PostgresWatermarkConnector(query);
|
|
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
const result = await connector.read({ partition_key: "dealer_stock_entry", limit: 2 });
|
|
expect(result.records.map((r) => r.record_key)).toEqual(["a1", "a2"]);
|
|
expect(result.exhausted).toBe(false); // 2 rows returned == limit, more may exist
|
|
});
|
|
|
|
it("a same-updated_at tie at the page boundary is not skipped on the next page (the whole reason for the tuple WHERE)", async () => {
|
|
const { query } = fixtureQuery(ROWS);
|
|
const connector = new PostgresWatermarkConnector(query);
|
|
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
const page1 = await connector.read({ partition_key: "dealer_stock_entry", limit: 1 }); // just a1
|
|
expect(page1.records.map((r) => r.record_key)).toEqual(["a1"]);
|
|
const page2 = await connector.read({ partition_key: "dealer_stock_entry", cursor: page1.next_cursor, limit: 1 });
|
|
// A naive `updated_at > $1` would skip a2 here (same updated_at as a1).
|
|
expect(page2.records.map((r) => r.record_key)).toEqual(["a2"]);
|
|
});
|
|
|
|
it("reads the full table across pages in strict watermark order, once per row", async () => {
|
|
const { query } = fixtureQuery(ROWS);
|
|
const connector = new PostgresWatermarkConnector(query);
|
|
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
let cursor: string | undefined;
|
|
const seen: string[] = [];
|
|
for (let i = 0; i < 10; i++) {
|
|
const page = await connector.read({ partition_key: "dealer_stock_entry", cursor, limit: 2 });
|
|
seen.push(...page.records.map((r) => r.record_key));
|
|
cursor = page.next_cursor;
|
|
if (page.exhausted) break;
|
|
}
|
|
expect(seen).toEqual(["a1", "a2", "a3", "a4"]);
|
|
});
|
|
|
|
it("the empty-table case: zero rows, no throw, exhausted:true", async () => {
|
|
const { query, calls } = fixtureQuery([]);
|
|
const connector = new PostgresWatermarkConnector(query);
|
|
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
const result = await connector.read({ partition_key: "dealer_stock_entry", limit: 100 });
|
|
expect(result.records).toEqual([]);
|
|
expect(result.exhausted).toBe(true);
|
|
expect(calls).toHaveLength(1); // the query WAS issued (a real SELECT against a real, currently-empty table) — it just returned nothing
|
|
});
|
|
});
|
|
|
|
describe("PostgresWatermarkConnector — checkpoint / resume", () => {
|
|
it("checkpoint() reflects the in-session cursor after a read", async () => {
|
|
const { query } = fixtureQuery(ROWS);
|
|
const connector = new PostgresWatermarkConnector(query);
|
|
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
await connector.read({ partition_key: "dealer_stock_entry", limit: 2 });
|
|
const cp = await connector.checkpoint("dealer_stock_entry");
|
|
const decoded = decodeCursor(cp.cursor);
|
|
expect(decoded?.last_id).toBe("a2");
|
|
expect(decoded?.last_updated_at).toBe("2026-09-01T00:00:00.000Z");
|
|
expect(decoded?.checksum).toBe(checksumRows(ROWS.slice(0, 2)));
|
|
});
|
|
|
|
it("resume() with a saved checkpoint continues exactly where the checkpoint left off", async () => {
|
|
const { query } = fixtureQuery(ROWS);
|
|
const connector = new PostgresWatermarkConnector(query);
|
|
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
const cursor = encodeCursor({ last_updated_at: "2026-09-02T00:00:00.000Z", last_id: "a3" });
|
|
const result = await connector.resume(
|
|
{ source_id: "s", tenant_id: "t", partition_key: "dealer_stock_entry", cursor, sequence: 3, records_committed: 3, committed_at: new Date().toISOString() },
|
|
10,
|
|
);
|
|
expect(result.records.map((r) => r.record_key)).toEqual(["a4"]);
|
|
});
|
|
});
|
|
|
|
describe("PostgresWatermarkConnector — real Postgres driver value shapes", () => {
|
|
it("a `timestamptz` column returned as a JS Date (not a string) is encoded as proper ISO-8601 in the cursor, not Date.toString()'s locale form", async () => {
|
|
// Regression: caught by running this connector against a REAL Postgres
|
|
// container (postgres.js hands back `timestamptz` as `Date`), not by any
|
|
// fixture-only test — the fixtures elsewhere in this file all use plain
|
|
// ISO strings, which never exercised this path. Reproduced here directly
|
|
// instead of requiring Docker so the regression stays covered in CI.
|
|
const rowsWithDates: SourceRow[] = [
|
|
{ id: "a1", vehicle_id: "v1", updated_at: new Date("2026-09-01T00:00:00.000Z") },
|
|
];
|
|
const { query } = fixtureQuery(rowsWithDates.map((r) => ({ ...r, updated_at: (r.updated_at as Date).toISOString() })));
|
|
// fixtureQuery's own row-compare logic needs string dates to sort; feed
|
|
// it a Date-returning query instead so the connector genuinely sees a
|
|
// Date object, the same as the real driver.
|
|
const dateReturningQuery: QueryImpl = async (sql, params) => {
|
|
const rows = await query(sql, params);
|
|
return rows.map((r) => ({ ...r, updated_at: new Date(r.updated_at as string) }));
|
|
};
|
|
const connector = new PostgresWatermarkConnector(dateReturningQuery);
|
|
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
const result = await connector.read({ partition_key: "dealer_stock_entry", limit: 10 });
|
|
expect(result.records[0].occurred_at).toBe("2026-09-01T00:00:00.000Z");
|
|
const decoded = decodeCursor(result.next_cursor);
|
|
expect(decoded?.last_updated_at).toBe("2026-09-01T00:00:00.000Z");
|
|
expect(decoded?.last_updated_at).not.toMatch(/GMT|\(.*\)/); // not Date.toString()'s form
|
|
});
|
|
});
|
|
|
|
describe("PostgresWatermarkConnector — never SELECT *", () => {
|
|
it("every SQL statement the connector builds names the configured columns explicitly", async () => {
|
|
const { query, calls } = fixtureQuery(ROWS);
|
|
const connector = new PostgresWatermarkConnector(query);
|
|
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
await connector.read({ partition_key: "dealer_stock_entry", limit: 10 });
|
|
for (const call of calls) {
|
|
expect(call.sql).not.toMatch(/select\s+\*/i);
|
|
expect(call.sql).toContain("id, vehicle_id, updated_at");
|
|
}
|
|
});
|
|
});
|