/** * BI-P012 DoD, against a LIVE ClickHouse: * * "Intentional query bomb is terminated without degrading unrelated tenant * beyond declared SLO." * * Two halves, and the second is the one that is usually skipped: * * 1. TERMINATED — a deliberately enormous query is refused by the server, * with the ClickHouse limit that stopped it identified by code. Each * limit BI-P012 names is fired individually, so "some limit stopped it" * is never mistaken for "the limit I claimed stopped it". * 2. UNRELATED TENANT NOT DEGRADED — a second tenant's interactive query is * timed WHILE the bomb is in flight and compared against a declared SLO, * not merely observed to "still work". */ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { ClickHouseAnalyticsEngine } from "@/src/analytics/engine/clickhouse-engine"; import { DEFAULT_BUDGETS } from "@/src/analytics/engine/resource-governance"; import type { AnalyticsIdentity } from "@/src/analytics/engine/siax-analytics-engine"; /** * Raw events carry a 30-day TTL (src/analytics/engine/schema.ts). Fixed * historical timestamps silently expire and make live-ClickHouse suites read * 0 rows, so seed rows are stamped relative to now (UTC, ClickHouse * DateTime64(3) text form). */ const SEED_EVENT_TIME = new Date(Date.now() - 2 * 3600_000).toISOString().replace("T", " ").replace("Z", ""); const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; const run = CLICKHOUSE_URL ? describe : describe.skip; const DB = `opt_gov_${Date.now().toString(36)}`; /** * The SLO this test asserts against. Declared as a constant so the claim is * falsifiable rather than "it still seemed to work". * * 5000 ms is deliberately generous for a 500-row lookup (it measures ~30 ms * on an idle server). It is set for the SHARED CI runner, not for a quiet * laptop: a tenant that had actually been starved by the bomb would take * seconds or fail outright, so the assertion still discriminates, while a * merely busy runner does not turn this gate into a flake. */ const UNRELATED_TENANT_SLO_MS = 5000; const VICTIM: AnalyticsIdentity = { tenant_id: "t_victim", project_id: "p_main", principal_id: "usr_v" }; let engine: ClickHouseAnalyticsEngine; /** Run a statement with explicit settings and report the ClickHouse code. */ async function chRun( sql: string, settings: Record = {}, body?: string, ): Promise<{ ok: boolean; code?: string; text: string; ms: number }> { const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); url.searchParams.set("query", sql); for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v); const headers: Record = {}; if (CLICKHOUSE_AUTH) headers["Authorization"] = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString("base64")}`; const t0 = Date.now(); const res = await fetch(url.toString(), { method: "POST", headers, body: body ?? "" }); const text = await res.text(); return { ok: res.ok, code: /Code: (\d+)/.exec(text)?.[1], text, ms: Date.now() - t0 }; } run("BI-P012 resource governance against a LIVE ClickHouse", () => { beforeAll(async () => { engine = new ClickHouseAnalyticsEngine({ baseUrl: CLICKHOUSE_URL!, basicAuth: CLICKHOUSE_AUTH, database: DB, dimensionCatalogs: { t_victim: { severity: "severity" } }, }); await engine.ensureSchema(); // A small, real dataset for the unrelated tenant. const rows = Array.from({ length: 500 }, (_, i) => JSON.stringify({ tenant_id: "t_victim", project_id: "p_main", dataset_id: "ds_v", metric: "victim_metric", event_id: `v${i}`, event_time: SEED_EVENT_TIME, value: 1, dim_names: ["severity"], dim_values: ["warning"], }), ).join("\n"); const ins = await chRun(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, {}, rows); expect(ins.ok, ins.text.slice(0, 200)).toBe(true); }, 120_000); afterAll(async () => { if (CLICKHOUSE_URL) await chRun(`DROP DATABASE IF EXISTS ${DB}`); }, 60_000); // ── each named limit, fired individually ──────────────────────────────── it("max_rows_to_read terminates a read bomb (CH code 158 TOO_MANY_ROWS)", async () => { const r = await chRun("SELECT count() FROM numbers(1000000000) FORMAT JSON", { readonly: "1", max_rows_to_read: "1000", read_overflow_mode: "throw", }); expect(r.ok).toBe(false); expect(r.code).toBe("158"); }); it("max_bytes_to_read terminates a scan bomb (CH code 307 TOO_MANY_BYTES)", async () => { const r = await chRun("SELECT sum(number) FROM numbers(1000000000) FORMAT JSON", { readonly: "1", max_bytes_to_read: "1024", read_overflow_mode: "throw", }); expect(r.ok).toBe(false); expect(r.code).toBe("307"); }); it("max_execution_time terminates a slow bomb (CH code 159 TIMEOUT_EXCEEDED)", async () => { const r = await chRun("SELECT count() FROM numbers(50000000000) FORMAT JSON", { readonly: "1", max_execution_time: "1", timeout_overflow_mode: "throw", }); expect(r.ok).toBe(false); expect(r.code).toBe("159"); expect(r.ms).toBeLessThan(20_000); }); it("max_result_rows terminates a result bomb and does NOT return a partial result", async () => { const r = await chRun("SELECT number FROM numbers(100000) FORMAT JSON", { readonly: "1", max_result_rows: "10", result_overflow_mode: "throw", }); expect(r.ok).toBe(false); expect(r.code).toBe("396"); // TOO_MANY_ROWS_OR_BYTES // The dangerous alternative: result_overflow_mode=break returns 10 rows // with HTTP 200 and no indication of truncation. const broken = await chRun("SELECT number FROM numbers(100000) FORMAT JSON", { readonly: "1", max_result_rows: "10", result_overflow_mode: "break", }); // MEASURED on ClickHouse 24.8: break mode answers HTTP 200 with 65409 of // 100000 rows (it stops at a block boundary, not at the stated 10) and // nothing in the payload says it is short. That is why every SELECT in // this engine ships result_overflow_mode=throw. expect(broken.ok).toBe(true); const brokenRows = (JSON.parse(broken.text) as { data: unknown[] }).data.length; expect(brokenRows).toBeGreaterThan(10); expect(brokenRows).toBeLessThan(100_000); }); it("max_memory_usage terminates a memory bomb (CH code 241 MEMORY_LIMIT_EXCEEDED)", async () => { const r = await chRun( "SELECT uniqExact(toString(number)) FROM numbers(50000000) FORMAT JSON", { readonly: "1", max_memory_usage: "1000000", max_execution_time: "30" }, ); expect(r.ok).toBe(false); expect(r.code).toBe("241"); }); it("readonly=1 means the query text cannot re-raise its own limits", async () => { const r = await chRun( "SELECT count() FROM numbers(1000000000) SETTINGS max_rows_to_read = 10000000000 FORMAT JSON", { readonly: "1", max_rows_to_read: "1000", read_overflow_mode: "throw" }, ); expect(r.ok).toBe(false); // 164 = READONLY (changing settings is forbidden); 158 = the limit still // applied. Either proves the caller did not win — but NOT a 200. expect(["164", "158"]).toContain(r.code); }); it("readonly=1 blocks writes through the query surface", async () => { const r = await chRun(`DROP TABLE ${DB}.opt_analytics_event FORMAT JSON`, { readonly: "1" }); expect(r.ok).toBe(false); }); // ── the half that is usually skipped ──────────────────────────────────── it("an unrelated tenant stays within its declared SLO WHILE a query bomb is in flight", async () => { // Baseline first, so "fast enough" is measured against this server, not // against a number someone hoped for. const baseline = await engine.executeMetric(VICTIM, { metric: "victim_metric", aggregate: "count" }); expect(baseline.rows[0].value).toBe(500); const baselineMs = baseline.stats.duration_ms; // Fire a genuine bomb the server must spend real CPU on. MEASURED: a bare // `count()` over numbers() is NOT a bomb — ClickHouse answers // numbers(20e9) in ~5s because it can skip the values entirely. Hashing // every row cannot be short-circuited, so this one really does burn CPU // until the limit kills it. const bomb = chRun("SELECT sum(cityHash64(number)) FROM numbers(20000000000) FORMAT JSON", { readonly: "1", max_execution_time: "8", timeout_overflow_mode: "throw", }); // While it burns, run the unrelated tenant's interactive query repeatedly. const samples: number[] = []; for (let i = 0; i < 5; i++) { const r = await engine.executeMetric(VICTIM, { metric: "victim_metric", aggregate: "count" }); expect(r.rows[0].value).toBe(500); // still CORRECT, not just fast samples.push(r.stats.duration_ms); } const bombResult = await bomb; expect(bombResult.ok, "the bomb was NOT terminated").toBe(false); expect(bombResult.code).toBe("159"); const worst = Math.max(...samples); expect( worst, `unrelated tenant degraded: baseline ${baselineMs}ms, worst under load ${worst}ms, SLO ${UNRELATED_TENANT_SLO_MS}ms`, ).toBeLessThan(UNRELATED_TENANT_SLO_MS); }, 60_000); // ── the local admission gate, on the real engine ──────────────────────── it("the per-tenant concurrency gate refuses the N+1st in-flight query for THAT tenant only", async () => { const limit = DEFAULT_BUDGETS.INTERACTIVE.maxConcurrentQueries; // Fire limit+4 concurrently; some must be refused with RATE_LIMITED // rather than all of them reaching the server. const attempts = Array.from({ length: limit + 4 }, () => engine .executeMetric(VICTIM, { metric: "victim_metric", aggregate: "count" }) .then(() => "ok" as const) .catch((e: { code?: string }) => e.code ?? "error"), ); const outcomes = await Promise.all(attempts); expect(outcomes.filter((o) => o === "RATE_LIMITED").length).toBeGreaterThan(0); expect(outcomes.filter((o) => o === "ok").length).toBeGreaterThan(0); // A different tenant is unaffected by the noisy one's exhausted slots. const other = await engine.executeMetric( { tenant_id: "t_quiet", project_id: "p_main", principal_id: "usr_q" }, { metric: "victim_metric", aggregate: "count" }, ); expect(other.rows[0].value).toBe(0); // its own (empty) data, and no error }, 60_000); });