Files
0pt/tests/connectors/postgres-keyed-lookup-connector.test.ts
T
Claude CodeandClaude Sonnet 5 07158099ee
CI (SIAX Cloud) / integration (push) Skipped
masterplan-lock / check (push) Successful in 7s
CI (SIAX Cloud) / security (push) Successful in 17s
masterplan-lock / check (pull_request) Successful in 9s
Security Scan (SIAX Cloud) / security (push) Successful in 38s
CI (SIAX Cloud) / security (pull_request) Successful in 26s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 26s
CI (SIAX Cloud) / quality (push) Successful in 57s
CI (SIAX Cloud) / integration (pull_request) Successful in 39s
CI (SIAX Cloud) / quality (pull_request) Successful in 59s
feat(ingestion): VehIQ OLTP -> ClickHouse watermark extractor (P070)
Implements Simon's specified pattern (dealer_stock_entry/deal ->
watermark/updated_at -> 0pt ingestion -> ClickHouse) as a P061 connector
pair (PostgresWatermarkConnector, PostgresKeyedLookupConnector) plugged
into the existing P062 IngestionRunner, writing to opt_dataset_projection
via SiaxAnalyticsEngine.ingestProjection.

Built here, not in admin/vehiq: that repo's own ADR-0035 (decisions 1/2/10,
CI-enforced by validate_engine_firewall.py) forbids VehIQ code from ever
holding a ClickHouse credential or hammering OLTP directly for analytics.
See docs/P070_VEHIQ_OLTP_EXTRACTOR.md for the full rationale.

Also corrects a real schema mismatch found while building this: production
has no generic `updated_at` on any of the three target tables (verified
live, read-only, 2026-09-25). dealer_stock_entry/deal use listed_at/
created_at as insert-time watermarks instead (documented caveat: in-place
updates are not caught without a schema change); app.listing has no
timestamp column at all and is deliberately not covered yet.

Vehicle dimension lookup is narrow by construction: keyed on exactly the
vehicle_id set referenced by the batch just read, never a scan of the
98M-row vehiq.vehicle table (connector-level validate() refuses that table
by name; runner-level tests assert the query only ever touches referenced
IDs).

Verified: fixture-backed connector/runner tests (watermark+limit,
same-timestamp tie at a page boundary, crash/resume, empty-table case,
narrow-lookup query-count assertions) + a real disposable Postgres
container (caught two real Date-handling bugs fixtures could not) + a
read-only proof against actual production (server10, vehiq-db): the
extractor's exact SQL returns zero rows, no error, confirming it is safe
to schedule with zero current effect.

.gitea/workflows/vehiq-oltp-ingest.yml is present, cron-scheduled, and
defaults to VEHIQ_INGEST_MODE=preview (read-only, never writes) with no
secrets configured — safe to merge with zero production effect until
explicitly activated per the doc's "what activating this requires".

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-25 19:11:15 +02:00

125 lines
5.6 KiB
TypeScript

/**
* Proves the narrow-scan property of PostgresKeyedLookupConnector — the
* connector that stands in for "dealer_stock_entry.vehicle_id -> unique
* vehicle IDs -> targeted lookup -> dimension_vehicle", never a scan of the
* 98M-row `vehicle` table. The assertions are on QUERY CALLS, not just
* returned records: a bug that widened the WHERE clause but happened to
* still return the right rows in a test fixture would be invisible to a
* behavioural-only test and is exactly what this file is written to catch.
*/
import { describe, expect, it } from "vitest";
import { PostgresKeyedLookupConnector } from "@/src/connectors/postgres-keyed-lookup-connector";
import type { QueryImpl, SourceRow } from "@/src/connectors/postgres-watermark-connector";
import type { RecordSchemaFields } from "@/src/schema/record-schema";
const SCHEMA: RecordSchemaFields = {
id: { type: "string", required: true },
make: { type: "string", required: false },
model: { type: "string", required: false },
};
const VEHICLE_TABLE: SourceRow[] = [
{ id: "v1", make: "Volvo", model: "XC60" },
{ id: "v2", make: "Volvo", model: "V70" },
{ id: "v3", make: "Toyota", model: "Corolla" },
{ id: "v4", make: "Audi", model: "A4" }, // never referenced by any dealer_stock_entry in these tests
];
function fixtureQuery(): { query: QueryImpl; calls: Array<{ sql: string; params: unknown[] }> } {
const calls: Array<{ sql: string; params: unknown[] }> = [];
const query: QueryImpl = async (sql, params) => {
calls.push({ sql, params });
const [ids] = params as [Array<string | number>];
return VEHICLE_TABLE.filter((row) => ids.includes(row.id as string));
};
return { query, calls };
}
function config(keys: Array<string | number>, keysPerQuery?: number) {
return {
table: "vehiq.vehicle",
partition_key: "dimension_vehicle",
columns: ["id", "make", "model"],
record_key_column: "id",
schema: SCHEMA,
keys,
keys_per_query: keysPerQuery,
};
}
describe("PostgresKeyedLookupConnector — validate()", () => {
it("refuses an empty key list rather than defaulting to a full read", () => {
const connector = new PostgresKeyedLookupConnector(fixtureQuery().query);
const verdict = connector.validate(config([]));
expect(verdict.valid).toBe(false);
expect(verdict.errors.join(" ")).toMatch(/keys must not be empty/);
});
it("rejects '*' in columns", () => {
const connector = new PostgresKeyedLookupConnector(fixtureQuery().query);
expect(connector.validate({ ...config(["v1"]), columns: ["*"] }).valid).toBe(false);
});
});
describe("PostgresKeyedLookupConnector — the narrow-scan property", () => {
it("connect() with an empty key list is refused, and NO query is ever issued", async () => {
const { query, calls } = fixtureQuery();
const connector = new PostgresKeyedLookupConnector(query);
await expect(
connector.connect(config([]), { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" }),
).rejects.toThrow(/keys must not be empty/);
expect(calls).toHaveLength(0);
});
it("reads exactly the vehicles referenced by the batch — v4 (never referenced) is never touched", async () => {
const referenced = ["v1", "v2", "v3"]; // simulates dedup'd dealer_stock_entry.vehicle_id values from one batch
const { query, calls } = fixtureQuery();
const connector = new PostgresKeyedLookupConnector(query);
await connector.connect(config(referenced), { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
const result = await connector.read({ partition_key: "dimension_vehicle", limit: 500 });
expect(result.records.map((r) => r.record_key).sort()).toEqual(["v1", "v2", "v3"]);
expect(result.exhausted).toBe(true);
// The narrow-scan property, checked at the QUERY level: every id ever
// bound into a query call is a subset of `referenced`, and the union of
// all ids queried equals exactly `referenced` — never more, never a
// wildcard, never "v4".
expect(calls).toHaveLength(1);
const allQueriedIds = calls.flatMap((c) => c.params[0] as string[]);
expect(allQueriedIds.sort()).toEqual([...referenced].sort());
expect(allQueriedIds).not.toContain("v4");
for (const call of calls) {
expect(call.sql).toMatch(/WHERE id = ANY\(\$1\)/);
expect(call.sql).not.toMatch(/select\s+\*/i);
}
});
it("chunks a large key list across multiple bounded queries, never one unbounded IN(...)", async () => {
const referenced = Array.from({ length: 5 }, (_, i) => `v${i + 1}`);
const { query, calls } = fixtureQuery();
const connector = new PostgresKeyedLookupConnector(query);
await connector.connect(config(referenced, 2), { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
let cursor: string | undefined;
let pages = 0;
for (let i = 0; i < 10; i++) {
const page = await connector.read({ partition_key: "dimension_vehicle", cursor, limit: 500 });
pages++;
cursor = page.next_cursor;
if (page.exhausted) break;
}
expect(pages).toBe(3); // ceil(5/2)
for (const call of calls) {
expect((call.params[0] as string[]).length).toBeLessThanOrEqual(2);
}
const allQueriedIds = calls.flatMap((c) => c.params[0] as string[]);
expect(new Set(allQueriedIds).size).toBe(5);
});
it("the empty-batch case: zero referenced vehicles means this connector is never even constructed by the caller (see vehiq-oltp-ingest-runner.test.ts) — but if it IS given zero keys, it fails validation rather than scanning", () => {
const connector = new PostgresKeyedLookupConnector(fixtureQuery().query);
expect(connector.validate(config([])).valid).toBe(false);
});
});