/** * Connector-level proof for postgres-watermark-connector.ts. No real * Postgres: `queryImpl` is a fixture that runs the SAME SQL text the real * connector builds against an in-memory row set, so the watermark math and * the tie-break are proven precisely rather than just "some rows came back". */ import { describe, expect, it } from "vitest"; import { PostgresWatermarkConnector, checksumRows, decodeCursor, encodeCursor, type QueryImpl, type SourceRow, } from "@/src/connectors/postgres-watermark-connector"; import type { RecordSchemaFields } from "@/src/schema/record-schema"; const SCHEMA: RecordSchemaFields = { id: { type: "string", required: true }, vehicle_id: { type: "string", required: false }, updated_at: { type: "string", required: true }, }; const CONFIG = { table: "app.dealer_stock_entry", partition_key: "dealer_stock_entry", columns: ["id", "vehicle_id", "updated_at"], record_key_column: "id", watermark_column: "updated_at", schema: SCHEMA, }; /** A fixture query engine that actually parses/applies the connector's WHERE/ORDER BY/LIMIT against an in-memory table — not a stub that ignores its arguments. */ function fixtureQuery(rows: SourceRow[]): { query: QueryImpl; calls: Array<{ sql: string; params: unknown[] }> } { const calls: Array<{ sql: string; params: unknown[] }> = []; const query: QueryImpl = async (sql, params) => { calls.push({ sql, params }); let filtered = rows; if (sql.includes("WHERE")) { const [watermark, id] = params as [string, string, number]; filtered = rows.filter( (r) => (r.updated_at as string) > watermark || ((r.updated_at as string) === watermark && (r.id as string) > id), ); } const sorted = [...filtered].sort((a, b) => { const w = (a.updated_at as string).localeCompare(b.updated_at as string); return w !== 0 ? w : (a.id as string).localeCompare(b.id as string); }); const limit = Number(params[params.length - 1]); return sorted.slice(0, limit); }; return { query, calls }; } const ROWS: SourceRow[] = [ { id: "a1", vehicle_id: "v1", updated_at: "2026-09-01T00:00:00.000Z" }, { id: "a2", vehicle_id: "v2", updated_at: "2026-09-01T00:00:00.000Z" }, // same updated_at as a1 — the tie-break case { id: "a3", vehicle_id: "v1", updated_at: "2026-09-02T00:00:00.000Z" }, { id: "a4", vehicle_id: "v3", updated_at: "2026-09-03T00:00:00.000Z" }, ]; describe("PostgresWatermarkConnector — validate()", () => { it("rejects a config pointed at the 98M-row vehicle table", () => { const connector = new PostgresWatermarkConnector(fixtureQuery([]).query); const verdict = connector.validate({ ...CONFIG, table: "vehiq.vehicle" }); expect(verdict.valid).toBe(false); expect(verdict.errors.join(" ")).toMatch(/vehicle/i); }); it("rejects SELECT * (an empty/missing columns list, or a literal '*')", () => { const connector = new PostgresWatermarkConnector(fixtureQuery([]).query); expect(connector.validate({ ...CONFIG, columns: [] }).valid).toBe(false); expect(connector.validate({ ...CONFIG, columns: ["*"] }).valid).toBe(false); }); it("accepts a well-formed config", () => { const connector = new PostgresWatermarkConnector(fixtureQuery([]).query); expect(connector.validate(CONFIG).valid).toBe(true); }); }); describe("PostgresWatermarkConnector — watermark + limit correctness", () => { it("first read (no cursor) honours LIMIT and orders by (updated_at, id)", async () => { const { query } = fixtureQuery(ROWS); const connector = new PostgresWatermarkConnector(query); await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" }); const result = await connector.read({ partition_key: "dealer_stock_entry", limit: 2 }); expect(result.records.map((r) => r.record_key)).toEqual(["a1", "a2"]); expect(result.exhausted).toBe(false); // 2 rows returned == limit, more may exist }); it("a same-updated_at tie at the page boundary is not skipped on the next page (the whole reason for the tuple WHERE)", async () => { const { query } = fixtureQuery(ROWS); const connector = new PostgresWatermarkConnector(query); await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" }); const page1 = await connector.read({ partition_key: "dealer_stock_entry", limit: 1 }); // just a1 expect(page1.records.map((r) => r.record_key)).toEqual(["a1"]); const page2 = await connector.read({ partition_key: "dealer_stock_entry", cursor: page1.next_cursor, limit: 1 }); // A naive `updated_at > $1` would skip a2 here (same updated_at as a1). expect(page2.records.map((r) => r.record_key)).toEqual(["a2"]); }); it("reads the full table across pages in strict watermark order, once per row", async () => { const { query } = fixtureQuery(ROWS); const connector = new PostgresWatermarkConnector(query); await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" }); let cursor: string | undefined; const seen: string[] = []; for (let i = 0; i < 10; i++) { const page = await connector.read({ partition_key: "dealer_stock_entry", cursor, limit: 2 }); seen.push(...page.records.map((r) => r.record_key)); cursor = page.next_cursor; if (page.exhausted) break; } expect(seen).toEqual(["a1", "a2", "a3", "a4"]); }); it("the empty-table case: zero rows, no throw, exhausted:true", async () => { const { query, calls } = fixtureQuery([]); const connector = new PostgresWatermarkConnector(query); await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" }); const result = await connector.read({ partition_key: "dealer_stock_entry", limit: 100 }); expect(result.records).toEqual([]); expect(result.exhausted).toBe(true); expect(calls).toHaveLength(1); // the query WAS issued (a real SELECT against a real, currently-empty table) — it just returned nothing }); }); describe("PostgresWatermarkConnector — checkpoint / resume", () => { it("checkpoint() reflects the in-session cursor after a read", async () => { const { query } = fixtureQuery(ROWS); const connector = new PostgresWatermarkConnector(query); await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" }); await connector.read({ partition_key: "dealer_stock_entry", limit: 2 }); const cp = await connector.checkpoint("dealer_stock_entry"); const decoded = decodeCursor(cp.cursor); expect(decoded?.last_id).toBe("a2"); expect(decoded?.last_updated_at).toBe("2026-09-01T00:00:00.000Z"); expect(decoded?.checksum).toBe(checksumRows(ROWS.slice(0, 2))); }); it("resume() with a saved checkpoint continues exactly where the checkpoint left off", async () => { const { query } = fixtureQuery(ROWS); const connector = new PostgresWatermarkConnector(query); await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" }); const cursor = encodeCursor({ last_updated_at: "2026-09-02T00:00:00.000Z", last_id: "a3" }); const result = await connector.resume( { source_id: "s", tenant_id: "t", partition_key: "dealer_stock_entry", cursor, sequence: 3, records_committed: 3, committed_at: new Date().toISOString() }, 10, ); expect(result.records.map((r) => r.record_key)).toEqual(["a4"]); }); }); describe("PostgresWatermarkConnector — real Postgres driver value shapes", () => { it("a `timestamptz` column returned as a JS Date (not a string) is encoded as proper ISO-8601 in the cursor, not Date.toString()'s locale form", async () => { // Regression: caught by running this connector against a REAL Postgres // container (postgres.js hands back `timestamptz` as `Date`), not by any // fixture-only test — the fixtures elsewhere in this file all use plain // ISO strings, which never exercised this path. Reproduced here directly // instead of requiring Docker so the regression stays covered in CI. const rowsWithDates: SourceRow[] = [ { id: "a1", vehicle_id: "v1", updated_at: new Date("2026-09-01T00:00:00.000Z") }, ]; const { query } = fixtureQuery(rowsWithDates.map((r) => ({ ...r, updated_at: (r.updated_at as Date).toISOString() }))); // fixtureQuery's own row-compare logic needs string dates to sort; feed // it a Date-returning query instead so the connector genuinely sees a // Date object, the same as the real driver. const dateReturningQuery: QueryImpl = async (sql, params) => { const rows = await query(sql, params); return rows.map((r) => ({ ...r, updated_at: new Date(r.updated_at as string) })); }; const connector = new PostgresWatermarkConnector(dateReturningQuery); await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" }); const result = await connector.read({ partition_key: "dealer_stock_entry", limit: 10 }); expect(result.records[0].occurred_at).toBe("2026-09-01T00:00:00.000Z"); const decoded = decodeCursor(result.next_cursor); expect(decoded?.last_updated_at).toBe("2026-09-01T00:00:00.000Z"); expect(decoded?.last_updated_at).not.toMatch(/GMT|\(.*\)/); // not Date.toString()'s form }); }); describe("PostgresWatermarkConnector — never SELECT *", () => { it("every SQL statement the connector builds names the configured columns explicitly", async () => { const { query, calls } = fixtureQuery(ROWS); const connector = new PostgresWatermarkConnector(query); await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" }); await connector.read({ partition_key: "dealer_stock_entry", limit: 10 }); for (const call of calls) { expect(call.sql).not.toMatch(/select\s+\*/i); expect(call.sql).toContain("id, vehicle_id, updated_at"); } }); });