CI (SIAX Cloud) / integration (push) Skipped
masterplan-lock / check (push) Successful in 7s
CI (SIAX Cloud) / security (push) Successful in 17s
masterplan-lock / check (pull_request) Successful in 9s
Security Scan (SIAX Cloud) / security (push) Successful in 38s
CI (SIAX Cloud) / security (pull_request) Successful in 26s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 26s
CI (SIAX Cloud) / quality (push) Successful in 57s
CI (SIAX Cloud) / integration (pull_request) Successful in 39s
CI (SIAX Cloud) / quality (pull_request) Successful in 59s
Implements Simon's specified pattern (dealer_stock_entry/deal -> watermark/updated_at -> 0pt ingestion -> ClickHouse) as a P061 connector pair (PostgresWatermarkConnector, PostgresKeyedLookupConnector) plugged into the existing P062 IngestionRunner, writing to opt_dataset_projection via SiaxAnalyticsEngine.ingestProjection. Built here, not in admin/vehiq: that repo's own ADR-0035 (decisions 1/2/10, CI-enforced by validate_engine_firewall.py) forbids VehIQ code from ever holding a ClickHouse credential or hammering OLTP directly for analytics. See docs/P070_VEHIQ_OLTP_EXTRACTOR.md for the full rationale. Also corrects a real schema mismatch found while building this: production has no generic `updated_at` on any of the three target tables (verified live, read-only, 2026-09-25). dealer_stock_entry/deal use listed_at/ created_at as insert-time watermarks instead (documented caveat: in-place updates are not caught without a schema change); app.listing has no timestamp column at all and is deliberately not covered yet. Vehicle dimension lookup is narrow by construction: keyed on exactly the vehicle_id set referenced by the batch just read, never a scan of the 98M-row vehiq.vehicle table (connector-level validate() refuses that table by name; runner-level tests assert the query only ever touches referenced IDs). Verified: fixture-backed connector/runner tests (watermark+limit, same-timestamp tie at a page boundary, crash/resume, empty-table case, narrow-lookup query-count assertions) + a real disposable Postgres container (caught two real Date-handling bugs fixtures could not) + a read-only proof against actual production (server10, vehiq-db): the extractor's exact SQL returns zero rows, no error, confirming it is safe to schedule with zero current effect. .gitea/workflows/vehiq-oltp-ingest.yml is present, cron-scheduled, and defaults to VEHIQ_INGEST_MODE=preview (read-only, never writes) with no secrets configured — safe to merge with zero production effect until explicitly activated per the doc's "what activating this requires". Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
195 lines
9.8 KiB
TypeScript
195 lines
9.8 KiB
TypeScript
/**
|
|
* Connector-level proof for postgres-watermark-connector.ts. No real
|
|
* Postgres: `queryImpl` is a fixture that runs the SAME SQL text the real
|
|
* connector builds against an in-memory row set, so the watermark math and
|
|
* the tie-break are proven precisely rather than just "some rows came back".
|
|
*/
|
|
import { describe, expect, it } from "vitest";
|
|
import {
|
|
PostgresWatermarkConnector,
|
|
checksumRows,
|
|
decodeCursor,
|
|
encodeCursor,
|
|
type QueryImpl,
|
|
type SourceRow,
|
|
} from "@/src/connectors/postgres-watermark-connector";
|
|
import type { RecordSchemaFields } from "@/src/schema/record-schema";
|
|
|
|
const SCHEMA: RecordSchemaFields = {
|
|
id: { type: "string", required: true },
|
|
vehicle_id: { type: "string", required: false },
|
|
updated_at: { type: "string", required: true },
|
|
};
|
|
|
|
const CONFIG = {
|
|
table: "app.dealer_stock_entry",
|
|
partition_key: "dealer_stock_entry",
|
|
columns: ["id", "vehicle_id", "updated_at"],
|
|
record_key_column: "id",
|
|
watermark_column: "updated_at",
|
|
schema: SCHEMA,
|
|
};
|
|
|
|
/** A fixture query engine that actually parses/applies the connector's WHERE/ORDER BY/LIMIT against an in-memory table — not a stub that ignores its arguments. */
|
|
function fixtureQuery(rows: SourceRow[]): { query: QueryImpl; calls: Array<{ sql: string; params: unknown[] }> } {
|
|
const calls: Array<{ sql: string; params: unknown[] }> = [];
|
|
const query: QueryImpl = async (sql, params) => {
|
|
calls.push({ sql, params });
|
|
let filtered = rows;
|
|
if (sql.includes("WHERE")) {
|
|
const [watermark, id] = params as [string, string, number];
|
|
filtered = rows.filter(
|
|
(r) => (r.updated_at as string) > watermark || ((r.updated_at as string) === watermark && (r.id as string) > id),
|
|
);
|
|
}
|
|
const sorted = [...filtered].sort((a, b) => {
|
|
const w = (a.updated_at as string).localeCompare(b.updated_at as string);
|
|
return w !== 0 ? w : (a.id as string).localeCompare(b.id as string);
|
|
});
|
|
const limit = Number(params[params.length - 1]);
|
|
return sorted.slice(0, limit);
|
|
};
|
|
return { query, calls };
|
|
}
|
|
|
|
const ROWS: SourceRow[] = [
|
|
{ id: "a1", vehicle_id: "v1", updated_at: "2026-09-01T00:00:00.000Z" },
|
|
{ id: "a2", vehicle_id: "v2", updated_at: "2026-09-01T00:00:00.000Z" }, // same updated_at as a1 — the tie-break case
|
|
{ id: "a3", vehicle_id: "v1", updated_at: "2026-09-02T00:00:00.000Z" },
|
|
{ id: "a4", vehicle_id: "v3", updated_at: "2026-09-03T00:00:00.000Z" },
|
|
];
|
|
|
|
describe("PostgresWatermarkConnector — validate()", () => {
|
|
it("rejects a config pointed at the 98M-row vehicle table", () => {
|
|
const connector = new PostgresWatermarkConnector(fixtureQuery([]).query);
|
|
const verdict = connector.validate({ ...CONFIG, table: "vehiq.vehicle" });
|
|
expect(verdict.valid).toBe(false);
|
|
expect(verdict.errors.join(" ")).toMatch(/vehicle/i);
|
|
});
|
|
|
|
it("rejects SELECT * (an empty/missing columns list, or a literal '*')", () => {
|
|
const connector = new PostgresWatermarkConnector(fixtureQuery([]).query);
|
|
expect(connector.validate({ ...CONFIG, columns: [] }).valid).toBe(false);
|
|
expect(connector.validate({ ...CONFIG, columns: ["*"] }).valid).toBe(false);
|
|
});
|
|
|
|
it("accepts a well-formed config", () => {
|
|
const connector = new PostgresWatermarkConnector(fixtureQuery([]).query);
|
|
expect(connector.validate(CONFIG).valid).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe("PostgresWatermarkConnector — watermark + limit correctness", () => {
|
|
it("first read (no cursor) honours LIMIT and orders by (updated_at, id)", async () => {
|
|
const { query } = fixtureQuery(ROWS);
|
|
const connector = new PostgresWatermarkConnector(query);
|
|
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
const result = await connector.read({ partition_key: "dealer_stock_entry", limit: 2 });
|
|
expect(result.records.map((r) => r.record_key)).toEqual(["a1", "a2"]);
|
|
expect(result.exhausted).toBe(false); // 2 rows returned == limit, more may exist
|
|
});
|
|
|
|
it("a same-updated_at tie at the page boundary is not skipped on the next page (the whole reason for the tuple WHERE)", async () => {
|
|
const { query } = fixtureQuery(ROWS);
|
|
const connector = new PostgresWatermarkConnector(query);
|
|
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
const page1 = await connector.read({ partition_key: "dealer_stock_entry", limit: 1 }); // just a1
|
|
expect(page1.records.map((r) => r.record_key)).toEqual(["a1"]);
|
|
const page2 = await connector.read({ partition_key: "dealer_stock_entry", cursor: page1.next_cursor, limit: 1 });
|
|
// A naive `updated_at > $1` would skip a2 here (same updated_at as a1).
|
|
expect(page2.records.map((r) => r.record_key)).toEqual(["a2"]);
|
|
});
|
|
|
|
it("reads the full table across pages in strict watermark order, once per row", async () => {
|
|
const { query } = fixtureQuery(ROWS);
|
|
const connector = new PostgresWatermarkConnector(query);
|
|
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
let cursor: string | undefined;
|
|
const seen: string[] = [];
|
|
for (let i = 0; i < 10; i++) {
|
|
const page = await connector.read({ partition_key: "dealer_stock_entry", cursor, limit: 2 });
|
|
seen.push(...page.records.map((r) => r.record_key));
|
|
cursor = page.next_cursor;
|
|
if (page.exhausted) break;
|
|
}
|
|
expect(seen).toEqual(["a1", "a2", "a3", "a4"]);
|
|
});
|
|
|
|
it("the empty-table case: zero rows, no throw, exhausted:true", async () => {
|
|
const { query, calls } = fixtureQuery([]);
|
|
const connector = new PostgresWatermarkConnector(query);
|
|
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
const result = await connector.read({ partition_key: "dealer_stock_entry", limit: 100 });
|
|
expect(result.records).toEqual([]);
|
|
expect(result.exhausted).toBe(true);
|
|
expect(calls).toHaveLength(1); // the query WAS issued (a real SELECT against a real, currently-empty table) — it just returned nothing
|
|
});
|
|
});
|
|
|
|
describe("PostgresWatermarkConnector — checkpoint / resume", () => {
|
|
it("checkpoint() reflects the in-session cursor after a read", async () => {
|
|
const { query } = fixtureQuery(ROWS);
|
|
const connector = new PostgresWatermarkConnector(query);
|
|
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
await connector.read({ partition_key: "dealer_stock_entry", limit: 2 });
|
|
const cp = await connector.checkpoint("dealer_stock_entry");
|
|
const decoded = decodeCursor(cp.cursor);
|
|
expect(decoded?.last_id).toBe("a2");
|
|
expect(decoded?.last_updated_at).toBe("2026-09-01T00:00:00.000Z");
|
|
expect(decoded?.checksum).toBe(checksumRows(ROWS.slice(0, 2)));
|
|
});
|
|
|
|
it("resume() with a saved checkpoint continues exactly where the checkpoint left off", async () => {
|
|
const { query } = fixtureQuery(ROWS);
|
|
const connector = new PostgresWatermarkConnector(query);
|
|
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
const cursor = encodeCursor({ last_updated_at: "2026-09-02T00:00:00.000Z", last_id: "a3" });
|
|
const result = await connector.resume(
|
|
{ source_id: "s", tenant_id: "t", partition_key: "dealer_stock_entry", cursor, sequence: 3, records_committed: 3, committed_at: new Date().toISOString() },
|
|
10,
|
|
);
|
|
expect(result.records.map((r) => r.record_key)).toEqual(["a4"]);
|
|
});
|
|
});
|
|
|
|
describe("PostgresWatermarkConnector — real Postgres driver value shapes", () => {
|
|
it("a `timestamptz` column returned as a JS Date (not a string) is encoded as proper ISO-8601 in the cursor, not Date.toString()'s locale form", async () => {
|
|
// Regression: caught by running this connector against a REAL Postgres
|
|
// container (postgres.js hands back `timestamptz` as `Date`), not by any
|
|
// fixture-only test — the fixtures elsewhere in this file all use plain
|
|
// ISO strings, which never exercised this path. Reproduced here directly
|
|
// instead of requiring Docker so the regression stays covered in CI.
|
|
const rowsWithDates: SourceRow[] = [
|
|
{ id: "a1", vehicle_id: "v1", updated_at: new Date("2026-09-01T00:00:00.000Z") },
|
|
];
|
|
const { query } = fixtureQuery(rowsWithDates.map((r) => ({ ...r, updated_at: (r.updated_at as Date).toISOString() })));
|
|
// fixtureQuery's own row-compare logic needs string dates to sort; feed
|
|
// it a Date-returning query instead so the connector genuinely sees a
|
|
// Date object, the same as the real driver.
|
|
const dateReturningQuery: QueryImpl = async (sql, params) => {
|
|
const rows = await query(sql, params);
|
|
return rows.map((r) => ({ ...r, updated_at: new Date(r.updated_at as string) }));
|
|
};
|
|
const connector = new PostgresWatermarkConnector(dateReturningQuery);
|
|
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
const result = await connector.read({ partition_key: "dealer_stock_entry", limit: 10 });
|
|
expect(result.records[0].occurred_at).toBe("2026-09-01T00:00:00.000Z");
|
|
const decoded = decodeCursor(result.next_cursor);
|
|
expect(decoded?.last_updated_at).toBe("2026-09-01T00:00:00.000Z");
|
|
expect(decoded?.last_updated_at).not.toMatch(/GMT|\(.*\)/); // not Date.toString()'s form
|
|
});
|
|
});
|
|
|
|
describe("PostgresWatermarkConnector — never SELECT *", () => {
|
|
it("every SQL statement the connector builds names the configured columns explicitly", async () => {
|
|
const { query, calls } = fixtureQuery(ROWS);
|
|
const connector = new PostgresWatermarkConnector(query);
|
|
await connector.connect(CONFIG, { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
await connector.read({ partition_key: "dealer_stock_entry", limit: 10 });
|
|
for (const call of calls) {
|
|
expect(call.sql).not.toMatch(/select\s+\*/i);
|
|
expect(call.sql).toContain("id, vehicle_id, updated_at");
|
|
}
|
|
});
|
|
});
|