CI (SIAX Cloud) / integration (push) Skipped
masterplan-lock / check (push) Successful in 7s
CI (SIAX Cloud) / security (push) Successful in 17s
masterplan-lock / check (pull_request) Successful in 9s
Security Scan (SIAX Cloud) / security (push) Successful in 38s
CI (SIAX Cloud) / security (pull_request) Successful in 26s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 26s
CI (SIAX Cloud) / quality (push) Successful in 57s
CI (SIAX Cloud) / integration (pull_request) Successful in 39s
CI (SIAX Cloud) / quality (pull_request) Successful in 59s
Implements Simon's specified pattern (dealer_stock_entry/deal -> watermark/updated_at -> 0pt ingestion -> ClickHouse) as a P061 connector pair (PostgresWatermarkConnector, PostgresKeyedLookupConnector) plugged into the existing P062 IngestionRunner, writing to opt_dataset_projection via SiaxAnalyticsEngine.ingestProjection. Built here, not in admin/vehiq: that repo's own ADR-0035 (decisions 1/2/10, CI-enforced by validate_engine_firewall.py) forbids VehIQ code from ever holding a ClickHouse credential or hammering OLTP directly for analytics. See docs/P070_VEHIQ_OLTP_EXTRACTOR.md for the full rationale. Also corrects a real schema mismatch found while building this: production has no generic `updated_at` on any of the three target tables (verified live, read-only, 2026-09-25). dealer_stock_entry/deal use listed_at/ created_at as insert-time watermarks instead (documented caveat: in-place updates are not caught without a schema change); app.listing has no timestamp column at all and is deliberately not covered yet. Vehicle dimension lookup is narrow by construction: keyed on exactly the vehicle_id set referenced by the batch just read, never a scan of the 98M-row vehiq.vehicle table (connector-level validate() refuses that table by name; runner-level tests assert the query only ever touches referenced IDs). Verified: fixture-backed connector/runner tests (watermark+limit, same-timestamp tie at a page boundary, crash/resume, empty-table case, narrow-lookup query-count assertions) + a real disposable Postgres container (caught two real Date-handling bugs fixtures could not) + a read-only proof against actual production (server10, vehiq-db): the extractor's exact SQL returns zero rows, no error, confirming it is safe to schedule with zero current effect. .gitea/workflows/vehiq-oltp-ingest.yml is present, cron-scheduled, and defaults to VEHIQ_INGEST_MODE=preview (read-only, never writes) with no secrets configured — safe to merge with zero production effect until explicitly activated per the doc's "what activating this requires". Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
125 lines
5.6 KiB
TypeScript
125 lines
5.6 KiB
TypeScript
/**
|
|
* Proves the narrow-scan property of PostgresKeyedLookupConnector — the
|
|
* connector that stands in for "dealer_stock_entry.vehicle_id -> unique
|
|
* vehicle IDs -> targeted lookup -> dimension_vehicle", never a scan of the
|
|
* 98M-row `vehicle` table. The assertions are on QUERY CALLS, not just
|
|
* returned records: a bug that widened the WHERE clause but happened to
|
|
* still return the right rows in a test fixture would be invisible to a
|
|
* behavioural-only test and is exactly what this file is written to catch.
|
|
*/
|
|
import { describe, expect, it } from "vitest";
|
|
import { PostgresKeyedLookupConnector } from "@/src/connectors/postgres-keyed-lookup-connector";
|
|
import type { QueryImpl, SourceRow } from "@/src/connectors/postgres-watermark-connector";
|
|
import type { RecordSchemaFields } from "@/src/schema/record-schema";
|
|
|
|
const SCHEMA: RecordSchemaFields = {
|
|
id: { type: "string", required: true },
|
|
make: { type: "string", required: false },
|
|
model: { type: "string", required: false },
|
|
};
|
|
|
|
const VEHICLE_TABLE: SourceRow[] = [
|
|
{ id: "v1", make: "Volvo", model: "XC60" },
|
|
{ id: "v2", make: "Volvo", model: "V70" },
|
|
{ id: "v3", make: "Toyota", model: "Corolla" },
|
|
{ id: "v4", make: "Audi", model: "A4" }, // never referenced by any dealer_stock_entry in these tests
|
|
];
|
|
|
|
function fixtureQuery(): { query: QueryImpl; calls: Array<{ sql: string; params: unknown[] }> } {
|
|
const calls: Array<{ sql: string; params: unknown[] }> = [];
|
|
const query: QueryImpl = async (sql, params) => {
|
|
calls.push({ sql, params });
|
|
const [ids] = params as [Array<string | number>];
|
|
return VEHICLE_TABLE.filter((row) => ids.includes(row.id as string));
|
|
};
|
|
return { query, calls };
|
|
}
|
|
|
|
function config(keys: Array<string | number>, keysPerQuery?: number) {
|
|
return {
|
|
table: "vehiq.vehicle",
|
|
partition_key: "dimension_vehicle",
|
|
columns: ["id", "make", "model"],
|
|
record_key_column: "id",
|
|
schema: SCHEMA,
|
|
keys,
|
|
keys_per_query: keysPerQuery,
|
|
};
|
|
}
|
|
|
|
describe("PostgresKeyedLookupConnector — validate()", () => {
|
|
it("refuses an empty key list rather than defaulting to a full read", () => {
|
|
const connector = new PostgresKeyedLookupConnector(fixtureQuery().query);
|
|
const verdict = connector.validate(config([]));
|
|
expect(verdict.valid).toBe(false);
|
|
expect(verdict.errors.join(" ")).toMatch(/keys must not be empty/);
|
|
});
|
|
|
|
it("rejects '*' in columns", () => {
|
|
const connector = new PostgresKeyedLookupConnector(fixtureQuery().query);
|
|
expect(connector.validate({ ...config(["v1"]), columns: ["*"] }).valid).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe("PostgresKeyedLookupConnector — the narrow-scan property", () => {
|
|
it("connect() with an empty key list is refused, and NO query is ever issued", async () => {
|
|
const { query, calls } = fixtureQuery();
|
|
const connector = new PostgresKeyedLookupConnector(query);
|
|
await expect(
|
|
connector.connect(config([]), { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" }),
|
|
).rejects.toThrow(/keys must not be empty/);
|
|
expect(calls).toHaveLength(0);
|
|
});
|
|
|
|
it("reads exactly the vehicles referenced by the batch — v4 (never referenced) is never touched", async () => {
|
|
const referenced = ["v1", "v2", "v3"]; // simulates dedup'd dealer_stock_entry.vehicle_id values from one batch
|
|
const { query, calls } = fixtureQuery();
|
|
const connector = new PostgresKeyedLookupConnector(query);
|
|
await connector.connect(config(referenced), { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
const result = await connector.read({ partition_key: "dimension_vehicle", limit: 500 });
|
|
|
|
expect(result.records.map((r) => r.record_key).sort()).toEqual(["v1", "v2", "v3"]);
|
|
expect(result.exhausted).toBe(true);
|
|
|
|
// The narrow-scan property, checked at the QUERY level: every id ever
|
|
// bound into a query call is a subset of `referenced`, and the union of
|
|
// all ids queried equals exactly `referenced` — never more, never a
|
|
// wildcard, never "v4".
|
|
expect(calls).toHaveLength(1);
|
|
const allQueriedIds = calls.flatMap((c) => c.params[0] as string[]);
|
|
expect(allQueriedIds.sort()).toEqual([...referenced].sort());
|
|
expect(allQueriedIds).not.toContain("v4");
|
|
for (const call of calls) {
|
|
expect(call.sql).toMatch(/WHERE id = ANY\(\$1\)/);
|
|
expect(call.sql).not.toMatch(/select\s+\*/i);
|
|
}
|
|
});
|
|
|
|
it("chunks a large key list across multiple bounded queries, never one unbounded IN(...)", async () => {
|
|
const referenced = Array.from({ length: 5 }, (_, i) => `v${i + 1}`);
|
|
const { query, calls } = fixtureQuery();
|
|
const connector = new PostgresKeyedLookupConnector(query);
|
|
await connector.connect(config(referenced, 2), { tenant_id: "t", project_id: "p", source_id: "s", run_id: "r" });
|
|
|
|
let cursor: string | undefined;
|
|
let pages = 0;
|
|
for (let i = 0; i < 10; i++) {
|
|
const page = await connector.read({ partition_key: "dimension_vehicle", cursor, limit: 500 });
|
|
pages++;
|
|
cursor = page.next_cursor;
|
|
if (page.exhausted) break;
|
|
}
|
|
expect(pages).toBe(3); // ceil(5/2)
|
|
for (const call of calls) {
|
|
expect((call.params[0] as string[]).length).toBeLessThanOrEqual(2);
|
|
}
|
|
const allQueriedIds = calls.flatMap((c) => c.params[0] as string[]);
|
|
expect(new Set(allQueriedIds).size).toBe(5);
|
|
});
|
|
|
|
it("the empty-batch case: zero referenced vehicles means this connector is never even constructed by the caller (see vehiq-oltp-ingest-runner.test.ts) — but if it IS given zero keys, it fails validation rather than scanning", () => {
|
|
const connector = new PostgresKeyedLookupConnector(fixtureQuery().query);
|
|
expect(connector.validate(config([])).valid).toBe(false);
|
|
});
|
|
});
|