Files
0pt/tests/connectors/postgres-watermark-connector.test.ts
T
Claude CodeandClaude Sonnet 5 07158099ee
CI (SIAX Cloud) / integration (push) Skipped
masterplan-lock / check (push) Successful in 7s
CI (SIAX Cloud) / security (push) Successful in 17s
masterplan-lock / check (pull_request) Successful in 9s
Security Scan (SIAX Cloud) / security (push) Successful in 38s
CI (SIAX Cloud) / security (pull_request) Successful in 26s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 26s
CI (SIAX Cloud) / quality (push) Successful in 57s
CI (SIAX Cloud) / integration (pull_request) Successful in 39s
CI (SIAX Cloud) / quality (pull_request) Successful in 59s
feat(ingestion): VehIQ OLTP -> ClickHouse watermark extractor (P070)
Implements Simon's specified pattern (dealer_stock_entry/deal ->
watermark/updated_at -> 0pt ingestion -> ClickHouse) as a P061 connector
pair (PostgresWatermarkConnector, PostgresKeyedLookupConnector) plugged
into the existing P062 IngestionRunner, writing to opt_dataset_projection
via SiaxAnalyticsEngine.ingestProjection.

Built here, not in admin/vehiq: that repo's own ADR-0035 (decisions 1/2/10,
CI-enforced by validate_engine_firewall.py) forbids VehIQ code from ever
holding a ClickHouse credential or hammering OLTP directly for analytics.
See docs/P070_VEHIQ_OLTP_EXTRACTOR.md for the full rationale.

Also corrects a real schema mismatch found while building this: production
has no generic `updated_at` on any of the three target tables (verified
live, read-only, 2026-09-25). dealer_stock_entry/deal use listed_at/
created_at as insert-time watermarks instead (documented caveat: in-place
updates are not caught without a schema change); app.listing has no
timestamp column at all and is deliberately not covered yet.

Vehicle dimension lookup is narrow by construction: keyed on exactly the
vehicle_id set referenced by the batch just read, never a scan of the
98M-row vehiq.vehicle table (connector-level validate() refuses that table
by name; runner-level tests assert the query only ever touches referenced
IDs).

Verified: fixture-backed connector/runner tests (watermark+limit,
same-timestamp tie at a page boundary, crash/resume, empty-table case,
narrow-lookup query-count assertions) + a real disposable Postgres
container (caught two real Date-handling bugs fixtures could not) + a
read-only proof against actual production (server10, vehiq-db): the
extractor's exact SQL returns zero rows, no error, confirming it is safe
to schedule with zero current effect.

.gitea/workflows/vehiq-oltp-ingest.yml is present, cron-scheduled, and
defaults to VEHIQ_INGEST_MODE=preview (read-only, never writes) with no
secrets configured — safe to merge with zero production effect until
explicitly activated per the doc's "what activating this requires".

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-25 19:11:15 +02:00

195 lines
9.8 KiB
TypeScript

/**
* Connector-level proof for postgres-watermark-connector.ts. No real
* Postgres: `queryImpl` is a fixture that runs the SAME SQL text the real
* connector builds against an in-memory row set, so the watermark math and
* the tie-break are proven precisely rather than just "some rows came back".
*/
import { describe, expect, it } from "vitest";
import {
PostgresWatermarkConnector,
checksumRows,
decodeCursor,
encodeCursor,
type QueryImpl,
type SourceRow,
} from "@/src/connectors/postgres-watermark-connector";
import type { RecordSchemaFields } from "@/src/schema/record-schema";
const SCHEMA: RecordSchemaFields = {
id: { type: "string", required: true },
vehicle_id: { type: "string", required: false },
updated_at: { type: "string", required: true },
};
const CONFIG = {
table: "app.dealer_stock_entry",
partition_key: "dealer_stock_entry",
columns: ["id", "vehicle_id", "updated_at"],
record_key_column: "id",
watermark_column: "updated_at",
schema: SCHEMA,
};
/** A fixture query engine that actually parses/applies the connector's WHERE/ORDER BY/LIMIT against an in-memory table — not a stub that ignores its arguments. */
function fixtureQuery(rows: SourceRow[]): { query: QueryImpl; calls: Array<{ sql: string; params: unknown[] }> } {
const calls: Array<{ sql: string; params: unknown[] }> = [];
const query: QueryImpl = async (sql, params) => {
calls.push({ sql, params });
let filtered = rows;
if (sql.includes("WHERE")) {
const [watermark, id] = params as [string, string, number];
filtered = rows.filter(
(r) => (r.updated_at as string) > watermark || ((r.updated_at as string) === watermark && (r.id as string) > id),
);
}
const sorted = [...filtered].sort((a, b) => {
const w = (a.updated_at as string).localeCompare(b.updated_at as string);
return w !== 0 ? w : (a.id as string).localeCompare(b.id as string);
});
const limit = Number(params[params.length - 1]);
return sorted.slice(0, limit);
};
return { query, calls };
}
const ROWS: SourceRow[] = [
{ id: "a1", vehicle_id: "v1", updated_at: "2026-09-01T00:00:00.000Z" },
{ id: "a2", vehicle_id: "v2", updated_at: "2026-09-01T00:00:00.000Z" }, // same updated_at as a1 — the tie-break case
{ id: "a3", vehicle_id: "v1", updated_at: "2026-09-02T00:00:00.000Z" },
{ id: "a4", vehicle_id: "v3", updated_at: "2026-09-03T00:00:00.000Z" },
];
describe("PostgresWatermarkConnector — validate()", () => {
it("rejects a config pointed at the 98M-row vehicle table", () => {
const connector = new PostgresWatermarkConnector(fixtureQuery([]).query);
const verdict = connector.validate({ ...CONFIG, table: "vehiq.vehicle" });
expect(verdict.valid).toBe(false);
expect(verdict.errors.join(" ")).toMatch(/vehicle/i);
});
it("rejects SELECT * (an empty/missing columns list, or a literal '*')", () => {
const connector = new PostgresWatermarkConnector(fixtureQuery([]).query);
expect(connector.validate({ ...CONFIG, columns: [] }).valid).toBe(false);
expect(connector.validate({ ...CONFIG, columns: ["*"] }).valid).toBe(false);
});
it("accepts a well-formed config", () => {
const connector = new PostgresWatermarkConnector(fixtureQuery([]).query);
expect(connector.validate(CONFIG).valid).toBe(true);
});
});
describe("PostgresWatermarkConnector — watermark + limit correctness", () => {
it("first read (no cursor) honours LIMIT and orders by (updated_at, id)", async () => {
const { query } = fixtureQuery(ROWS);
const connector = new PostgresWatermarkConnector(query);
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
const result = await connector.read({ partition_key: "dealer_stock_entry", limit: 2 });
expect(result.records.map((r) => r.record_key)).toEqual(["a1", "a2"]);
expect(result.exhausted).toBe(false); // 2 rows returned == limit, more may exist
});
it("a same-updated_at tie at the page boundary is not skipped on the next page (the whole reason for the tuple WHERE)", async () => {
const { query } = fixtureQuery(ROWS);
const connector = new PostgresWatermarkConnector(query);
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
const page1 = await connector.read({ partition_key: "dealer_stock_entry", limit: 1 }); // just a1
expect(page1.records.map((r) => r.record_key)).toEqual(["a1"]);
const page2 = await connector.read({ partition_key: "dealer_stock_entry", cursor: page1.next_cursor, limit: 1 });
// A naive `updated_at > $1` would skip a2 here (same updated_at as a1).
expect(page2.records.map((r) => r.record_key)).toEqual(["a2"]);
});
it("reads the full table across pages in strict watermark order, once per row", async () => {
const { query } = fixtureQuery(ROWS);
const connector = new PostgresWatermarkConnector(query);
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
let cursor: string | undefined;
const seen: string[] = [];
for (let i = 0; i < 10; i++) {
const page = await connector.read({ partition_key: "dealer_stock_entry", cursor, limit: 2 });
seen.push(...page.records.map((r) => r.record_key));
cursor = page.next_cursor;
if (page.exhausted) break;
}
expect(seen).toEqual(["a1", "a2", "a3", "a4"]);
});
it("the empty-table case: zero rows, no throw, exhausted:true", async () => {
const { query, calls } = fixtureQuery([]);
const connector = new PostgresWatermarkConnector(query);
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
const result = await connector.read({ partition_key: "dealer_stock_entry", limit: 100 });
expect(result.records).toEqual([]);
expect(result.exhausted).toBe(true);
expect(calls).toHaveLength(1); // the query WAS issued (a real SELECT against a real, currently-empty table) — it just returned nothing
});
});
describe("PostgresWatermarkConnector — checkpoint / resume", () => {
it("checkpoint() reflects the in-session cursor after a read", async () => {
const { query } = fixtureQuery(ROWS);
const connector = new PostgresWatermarkConnector(query);
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
await connector.read({ partition_key: "dealer_stock_entry", limit: 2 });
const cp = await connector.checkpoint("dealer_stock_entry");
const decoded = decodeCursor(cp.cursor);
expect(decoded?.last_id).toBe("a2");
expect(decoded?.last_updated_at).toBe("2026-09-01T00:00:00.000Z");
expect(decoded?.checksum).toBe(checksumRows(ROWS.slice(0, 2)));
});
it("resume() with a saved checkpoint continues exactly where the checkpoint left off", async () => {
const { query } = fixtureQuery(ROWS);
const connector = new PostgresWatermarkConnector(query);
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
const cursor = encodeCursor({ last_updated_at: "2026-09-02T00:00:00.000Z", last_id: "a3" });
const result = await connector.resume(
{ source_id: "s", tenant_id: "t", partition_key: "dealer_stock_entry", cursor, sequence: 3, records_committed: 3, committed_at: new Date().toISOString() },
10,
);
expect(result.records.map((r) => r.record_key)).toEqual(["a4"]);
});
});
describe("PostgresWatermarkConnector — real Postgres driver value shapes", () => {
it("a `timestamptz` column returned as a JS Date (not a string) is encoded as proper ISO-8601 in the cursor, not Date.toString()'s locale form", async () => {
// Regression: caught by running this connector against a REAL Postgres
// container (postgres.js hands back `timestamptz` as `Date`), not by any
// fixture-only test — the fixtures elsewhere in this file all use plain
// ISO strings, which never exercised this path. Reproduced here directly
// instead of requiring Docker so the regression stays covered in CI.
const rowsWithDates: SourceRow[] = [
{ id: "a1", vehicle_id: "v1", updated_at: new Date("2026-09-01T00:00:00.000Z") },
];
const { query } = fixtureQuery(rowsWithDates.map((r) => ({ ...r, updated_at: (r.updated_at as Date).toISOString() })));
// fixtureQuery's own row-compare logic needs string dates to sort; feed
// it a Date-returning query instead so the connector genuinely sees a
// Date object, the same as the real driver.
const dateReturningQuery: QueryImpl = async (sql, params) => {
const rows = await query(sql, params);
return rows.map((r) => ({ ...r, updated_at: new Date(r.updated_at as string) }));
};
const connector = new PostgresWatermarkConnector(dateReturningQuery);
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
const result = await connector.read({ partition_key: "dealer_stock_entry", limit: 10 });
expect(result.records[0].occurred_at).toBe("2026-09-01T00:00:00.000Z");
const decoded = decodeCursor(result.next_cursor);
expect(decoded?.last_updated_at).toBe("2026-09-01T00:00:00.000Z");
expect(decoded?.last_updated_at).not.toMatch(/GMT|\(.*\)/); // not Date.toString()'s form
});
});
describe("PostgresWatermarkConnector — never SELECT *", () => {
it("every SQL statement the connector builds names the configured columns explicitly", async () => {
const { query, calls } = fixtureQuery(ROWS);
const connector = new PostgresWatermarkConnector(query);
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
await connector.read({ partition_key: "dealer_stock_entry", limit: 10 });
for (const call of calls) {
expect(call.sql).not.toMatch(/select\s+\*/i);
expect(call.sql).toContain("id, vehicle_id, updated_at");
}
});
});