Files
0pt/tests/connectors/postgres-watermark-connector.test.ts
T

195 lines
9.8 KiB
TypeScript

/**
* Connector-level proof for postgres-watermark-connector.ts. No real
* Postgres: `queryImpl` is a fixture that runs the SAME SQL text the real
* connector builds against an in-memory row set, so the watermark math and
* the tie-break are proven precisely rather than just "some rows came back".
*/
import { describe, expect, it } from "vitest";
import {
PostgresWatermarkConnector,
checksumRows,
decodeCursor,
encodeCursor,
type QueryImpl,
type SourceRow,
} from "@/src/connectors/postgres-watermark-connector";
import type { RecordSchemaFields } from "@/src/schema/record-schema";
const SCHEMA: RecordSchemaFields = {
id: { type: "string", required: true },
vehicle_id: { type: "string", required: false },
updated_at: { type: "string", required: true },
};
const CONFIG = {
table: "app.dealer_stock_entry",
partition_key: "dealer_stock_entry",
columns: ["id", "vehicle_id", "updated_at"],
record_key_column: "id",
watermark_column: "updated_at",
schema: SCHEMA,
};
/** A fixture query engine that actually parses/applies the connector's WHERE/ORDER BY/LIMIT against an in-memory table — not a stub that ignores its arguments. */
function fixtureQuery(rows: SourceRow[]): { query: QueryImpl; calls: Array<{ sql: string; params: unknown[] }> } {
const calls: Array<{ sql: string; params: unknown[] }> = [];
const query: QueryImpl = async (sql, params) => {
calls.push({ sql, params });
let filtered = rows;
if (sql.includes("WHERE")) {
const [watermark, id] = params as [string, string, number];
filtered = rows.filter(
(r) => (r.updated_at as string) > watermark || ((r.updated_at as string) === watermark && (r.id as string) > id),
);
}
const sorted = [...filtered].sort((a, b) => {
const w = (a.updated_at as string).localeCompare(b.updated_at as string);
return w !== 0 ? w : (a.id as string).localeCompare(b.id as string);
});
const limit = Number(params[params.length - 1]);
return sorted.slice(0, limit);
};
return { query, calls };
}
const ROWS: SourceRow[] = [
{ id: "a1", vehicle_id: "v1", updated_at: "2026-09-01T00:00:00.000Z" },
{ id: "a2", vehicle_id: "v2", updated_at: "2026-09-01T00:00:00.000Z" }, // same updated_at as a1 — the tie-break case
{ id: "a3", vehicle_id: "v1", updated_at: "2026-09-02T00:00:00.000Z" },
{ id: "a4", vehicle_id: "v3", updated_at: "2026-09-03T00:00:00.000Z" },
];
describe("PostgresWatermarkConnector — validate()", () => {
it("rejects a config pointed at the 98M-row vehicle table", () => {
const connector = new PostgresWatermarkConnector(fixtureQuery([]).query);
const verdict = connector.validate({ ...CONFIG, table: "vehiq.vehicle" });
expect(verdict.valid).toBe(false);
expect(verdict.errors.join(" ")).toMatch(/vehicle/i);
});
it("rejects SELECT * (an empty/missing columns list, or a literal '*')", () => {
const connector = new PostgresWatermarkConnector(fixtureQuery([]).query);
expect(connector.validate({ ...CONFIG, columns: [] }).valid).toBe(false);
expect(connector.validate({ ...CONFIG, columns: ["*"] }).valid).toBe(false);
});
it("accepts a well-formed config", () => {
const connector = new PostgresWatermarkConnector(fixtureQuery([]).query);
expect(connector.validate(CONFIG).valid).toBe(true);
});
});
describe("PostgresWatermarkConnector — watermark + limit correctness", () => {
it("first read (no cursor) honours LIMIT and orders by (updated_at, id)", async () => {
const { query } = fixtureQuery(ROWS);
const connector = new PostgresWatermarkConnector(query);
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
const result = await connector.read({ partition_key: "dealer_stock_entry", limit: 2 });
expect(result.records.map((r) => r.record_key)).toEqual(["a1", "a2"]);
expect(result.exhausted).toBe(false); // 2 rows returned == limit, more may exist
});
it("a same-updated_at tie at the page boundary is not skipped on the next page (the whole reason for the tuple WHERE)", async () => {
const { query } = fixtureQuery(ROWS);
const connector = new PostgresWatermarkConnector(query);
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
const page1 = await connector.read({ partition_key: "dealer_stock_entry", limit: 1 }); // just a1
expect(page1.records.map((r) => r.record_key)).toEqual(["a1"]);
const page2 = await connector.read({ partition_key: "dealer_stock_entry", cursor: page1.next_cursor, limit: 1 });
// A naive `updated_at > $1` would skip a2 here (same updated_at as a1).
expect(page2.records.map((r) => r.record_key)).toEqual(["a2"]);
});
it("reads the full table across pages in strict watermark order, once per row", async () => {
const { query } = fixtureQuery(ROWS);
const connector = new PostgresWatermarkConnector(query);
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
let cursor: string | undefined;
const seen: string[] = [];
for (let i = 0; i < 10; i++) {
const page = await connector.read({ partition_key: "dealer_stock_entry", cursor, limit: 2 });
seen.push(...page.records.map((r) => r.record_key));
cursor = page.next_cursor;
if (page.exhausted) break;
}
expect(seen).toEqual(["a1", "a2", "a3", "a4"]);
});
it("the empty-table case: zero rows, no throw, exhausted:true", async () => {
const { query, calls } = fixtureQuery([]);
const connector = new PostgresWatermarkConnector(query);
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
const result = await connector.read({ partition_key: "dealer_stock_entry", limit: 100 });
expect(result.records).toEqual([]);
expect(result.exhausted).toBe(true);
expect(calls).toHaveLength(1); // the query WAS issued (a real SELECT against a real, currently-empty table) — it just returned nothing
});
});
describe("PostgresWatermarkConnector — checkpoint / resume", () => {
it("checkpoint() reflects the in-session cursor after a read", async () => {
const { query } = fixtureQuery(ROWS);
const connector = new PostgresWatermarkConnector(query);
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
await connector.read({ partition_key: "dealer_stock_entry", limit: 2 });
const cp = await connector.checkpoint("dealer_stock_entry");
const decoded = decodeCursor(cp.cursor);
expect(decoded?.last_id).toBe("a2");
expect(decoded?.last_updated_at).toBe("2026-09-01T00:00:00.000Z");
expect(decoded?.checksum).toBe(checksumRows(ROWS.slice(0, 2)));
});
it("resume() with a saved checkpoint continues exactly where the checkpoint left off", async () => {
const { query } = fixtureQuery(ROWS);
const connector = new PostgresWatermarkConnector(query);
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
const cursor = encodeCursor({ last_updated_at: "2026-09-02T00:00:00.000Z", last_id: "a3" });
const result = await connector.resume(
{ source_id: "s", tenant_id: "t", partition_key: "dealer_stock_entry", cursor, sequence: 3, records_committed: 3, committed_at: new Date().toISOString() },
10,
);
expect(result.records.map((r) => r.record_key)).toEqual(["a4"]);
});
});
describe("PostgresWatermarkConnector — real Postgres driver value shapes", () => {
it("a `timestamptz` column returned as a JS Date (not a string) is encoded as proper ISO-8601 in the cursor, not Date.toString()'s locale form", async () => {
// Regression: caught by running this connector against a REAL Postgres
// container (postgres.js hands back `timestamptz` as `Date`), not by any
// fixture-only test — the fixtures elsewhere in this file all use plain
// ISO strings, which never exercised this path. Reproduced here directly
// instead of requiring Docker so the regression stays covered in CI.
const rowsWithDates: SourceRow[] = [
{ id: "a1", vehicle_id: "v1", updated_at: new Date("2026-09-01T00:00:00.000Z") },
];
const { query } = fixtureQuery(rowsWithDates.map((r) => ({ ...r, updated_at: (r.updated_at as Date).toISOString() })));
// fixtureQuery's own row-compare logic needs string dates to sort; feed
// it a Date-returning query instead so the connector genuinely sees a
// Date object, the same as the real driver.
const dateReturningQuery: QueryImpl = async (sql, params) => {
const rows = await query(sql, params);
return rows.map((r) => ({ ...r, updated_at: new Date(r.updated_at as string) }));
};
const connector = new PostgresWatermarkConnector(dateReturningQuery);
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
const result = await connector.read({ partition_key: "dealer_stock_entry", limit: 10 });
expect(result.records[0].occurred_at).toBe("2026-09-01T00:00:00.000Z");
const decoded = decodeCursor(result.next_cursor);
expect(decoded?.last_updated_at).toBe("2026-09-01T00:00:00.000Z");
expect(decoded?.last_updated_at).not.toMatch(/GMT|\(.*\)/); // not Date.toString()'s form
});
});
describe("PostgresWatermarkConnector — never SELECT *", () => {
it("every SQL statement the connector builds names the configured columns explicitly", async () => {
const { query, calls } = fixtureQuery(ROWS);
const connector = new PostgresWatermarkConnector(query);
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
await connector.read({ partition_key: "dealer_stock_entry", limit: 10 });
for (const call of calls) {
expect(call.sql).not.toMatch(/select\s+\*/i);
expect(call.sql).toContain("id, vehicle_id, updated_at");
}
});
});