/** * tests/integration/query-cancellation-budget.integration.test.ts * ================================================================ * BI-P019 (cancellation) and BI-P021 (cost/budget) against a live ClickHouse. * * Both prompts are only real if something ACTUALLY STOPS. So both are tested * by starting a genuine CPU bomb and requiring the server to stop working: * * * the query must disappear from `system.processes`; * * `system.query_log` must record it as ExceptionWhileProcessing with * ClickHouse code 394 (QUERY_WAS_CANCELLED) — the server's own account of * what happened, not ours; * * and the COUNTERFACTUAL must hold: an identical bomb left alone for the * same interval is still running. Without that, "the query was gone after * we cancelled it" is equally consistent with "the query was short". * * BI-P021's in-flight enforcement gets the same treatment: a tenant window * budget of one million rows, a query that scans far more, and the requirement * that the enforcer cancels it MID-FLIGHT with `rows_read` already non-zero — * i.e. it was running, it was observed running, and it was stopped. * * Gated on CLICKHOUSE_URL. With the variable set and the server down, this * suite FAILS; it does not degrade to a skip. */ import fs from 'node:fs'; import path from 'node:path'; import { beforeAll, describe, expect, it } from 'vitest'; import { ClickHouseProcessControl } from '@/src/analytics/execution/clickhouse-process-control'; import { cancelAndVerify, cancelOrThrow } from '@/src/analytics/execution/cancellation'; import { DEFAULT_QUERY_BUDGET, InMemoryWindowStore, QueryBudgetEnforcer, } from '@/src/analytics/cost/query-budget'; const BASE_URL = process.env.CLICKHOUSE_URL ?? ''; const AUTH = process.env.CLICKHOUSE_AUTH ?? 'default:'; const suite = BASE_URL ? describe : describe.skip; const EVIDENCE_PATH = path.join(process.cwd(), 'evidence/bi-p019-p021/cancellation-and-budget.json'); /** Big enough that it cannot finish inside any of the windows below. */ const BOMB = 'SELECT sum(number) AS s FROM numbers_mt(200000000000)'; interface Res { ok: boolean; status: number; body: string; } async function send(sql: string, opts: { queryId?: string; timeoutMs?: number } = {}): Promise { const url = new URL(BASE_URL); url.searchParams.set('query', sql); if (opts.queryId) url.searchParams.set('query_id', opts.queryId); try { const res = await fetch(url.toString(), { method: 'POST', headers: { Authorization: `Basic ${Buffer.from(AUTH).toString('base64')}` }, body: '', signal: AbortSignal.timeout(opts.timeoutMs ?? 120_000), }); return { ok: res.ok, status: res.status, body: await res.text() }; } catch (e) { return { ok: false, status: 0, body: String(e) }; } } /** Fire and forget: the bomb runs on the server while the test carries on. */ function launch(queryId: string): Promise { return send(BOMB, { queryId, timeoutMs: 120_000 }); } const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); const control = new ClickHouseProcessControl({ baseUrl: BASE_URL || 'http://unset', basicAuth: AUTH }); const evidence: Record = {}; async function waitUntilRunning(queryId: string, timeoutMs = 10_000): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { if (await control.isRunning(queryId)) return true; await sleep(50); } return false; } /** ClickHouse's own verdict, from its query log. 394 = QUERY_WAS_CANCELLED. */ async function queryLogVerdict( queryId: string, ): Promise<{ type: string; exception_code: number } | undefined> { await send('SYSTEM FLUSH LOGS'); const res = await send( `SELECT type, exception_code FROM system.query_log WHERE query_id = '${queryId}' ` + `AND type != 'QueryStart' ORDER BY event_time_microseconds DESC LIMIT 1 FORMAT JSON`, ); if (!res.ok) return undefined; const row = (JSON.parse(res.body) as { data: Array<{ type: string; exception_code: number | string }> }) .data[0]; return row ? { type: row.type, exception_code: Number(row.exception_code) } : undefined; } beforeAll(async () => { if (!BASE_URL) return; const v = await send('SELECT version()'); if (!v.ok) throw new Error(`CLICKHOUSE_URL is set but ClickHouse is not answering: ${v.status}`); evidence.clickhouse_version = v.body.trim(); evidence.recorded_at = new Date().toISOString(); }, 60_000); suite('BI-P019 — cancellation stops an expensive query, and it is VERIFIED', () => { it('counterfactual: an identical bomb left alone is still running after the same interval', async () => { const queryId = `opt_cf_${Date.now()}`; void launch(queryId); expect(await waitUntilRunning(queryId)).toBe(true); await sleep(2_000); const stillRunning = await control.isRunning(queryId); evidence.counterfactual_still_running_after_2s = stillRunning; expect(stillRunning, 'the bomb finished on its own — it is not expensive enough to prove anything').toBe( true, ); // Clean up, and incidentally exercise the same path. await cancelAndVerify(control, queryId, { timeoutMs: 10_000 }); }, 90_000); it("cancellation is observed to stop it, and ClickHouse's own log agrees", async () => { const queryId = `opt_cancel_${Date.now()}`; void launch(queryId); expect(await waitUntilRunning(queryId)).toBe(true); // Let it accumulate real work so "it stopped" is a statement about a // running query, not about one that had barely begun. await sleep(1_000); const progressBefore = await control.progress(queryId); expect(progressBefore?.rows_read ?? 0).toBeGreaterThan(0); const outcome = await cancelAndVerify(control, queryId, { timeoutMs: 10_000, pollIntervalMs: 100 }); expect(outcome.kill_accepted).toBe(true); expect(outcome.verified_stopped, 'the query outlived its cancellation deadline').toBe(true); expect(outcome.already_finished).toBe(false); expect(outcome.elapsed_ms).toBeLessThan(10_000); expect(await control.isRunning(queryId)).toBe(false); const verdict = await queryLogVerdict(queryId); evidence.cancellation = { rows_read_before_cancel: progressBefore?.rows_read, elapsed_ms_to_verified_stop: outcome.elapsed_ms, verified_stopped: outcome.verified_stopped, clickhouse_query_log_type: verdict?.type, clickhouse_exception_code: verdict?.exception_code, }; // 394 = QUERY_WAS_CANCELLED. The server saying it, not us. expect(verdict?.exception_code, `query_log verdict was ${JSON.stringify(verdict)}`).toBe(394); }, 90_000); it('cancelling a query that already finished reports already_finished, not a false stop', async () => { const queryId = `opt_done_${Date.now()}`; const res = await send('SELECT 1', { queryId }); expect(res.ok).toBe(true); const outcome = await cancelAndVerify(control, queryId, { timeoutMs: 2_000 }); expect(outcome.already_finished).toBe(true); expect(outcome.verified_stopped).toBe(true); }); it('cancelOrThrow raises when the query is not observed to stop in time', async () => { const queryId = `opt_deadline_${Date.now()}`; void launch(queryId); expect(await waitUntilRunning(queryId)).toBe(true); // A deadline shorter than ClickHouse can plausibly unwind in, with the // kill request suppressed so the query genuinely keeps running. The // point is that the module REPORTS a failure to stop rather than // returning a cheerful boolean. const noKill = { engine: 'clickhouse', isRunning: (id: string) => control.isRunning(id), requestKill: async () => false, progress: (id: string) => control.progress(id), }; await expect(cancelOrThrow(noKill, queryId, { timeoutMs: 300, pollIntervalMs: 50 })).rejects.toThrow( /not observed to stop/, ); await cancelAndVerify(control, queryId, { timeoutMs: 10_000 }); }, 90_000); }); suite('BI-P021 — the budget actually stops a running query', () => { it('an in-flight window breach cancels the query mid-scan, with the cancellation verified', async () => { const store = new InMemoryWindowStore(); const settled: unknown[] = []; const enforcer = new QueryBudgetEnforcer({ budget: { ...DEFAULT_QUERY_BUDGET, maxRowsScanned: 1_000_000, maxCostUnits: 1e9 }, store, ledger: { record: (e) => void settled.push(e) }, control, enforceInFlight: true, pollIntervalMs: 100, storeIsShared: false, }); const queryId = `opt_budget_${Date.now()}`; void launch(queryId); expect(await waitUntilRunning(queryId)).toBe(true); const outcome = await enforcer.enforceInFlight('t_alpha', queryId); expect(outcome.terminated, 'the budget watched the query run past its window and did nothing').toBe(true); expect(outcome.reason).toBe('rows'); // It was RUNNING when it was stopped — not stopped before it started. expect(outcome.observed.rows_read).toBeGreaterThan(1_000_000); expect(outcome.cancellation?.verified_stopped).toBe(true); expect(await control.isRunning(queryId)).toBe(false); const verdict = await queryLogVerdict(queryId); expect(verdict?.exception_code).toBe(394); await enforcer.settle({ tenant_id: 't_alpha', principal_id: 'u_1', query_id: queryId, cost: { rows_scanned: outcome.observed.rows_read, bytes_scanned: outcome.observed.bytes_read, execution_ms: outcome.observed.elapsed_ms, engine: 'clickhouse', }, terminated_by_budget: true, }); expect(settled).toHaveLength(1); evidence.budget_in_flight = { budget_max_rows_scanned: 1_000_000, observed_rows_read_at_cancel: outcome.observed.rows_read, observed_elapsed_ms_at_cancel: outcome.observed.elapsed_ms, polls_before_breach: outcome.polls, overshoot_factor: Math.round(outcome.observed.rows_read / 1_000_000), overshoot_note: 'in-flight enforcement is granular to the poll interval; the query overshot the window ' + 'by this factor before the poll that caught it. A bound, not a hard cap.', cancellation_verified: outcome.cancellation?.verified_stopped, clickhouse_exception_code: verdict?.exception_code, charged_for_work_done_before_cancel: true, }; }, 120_000); it('admission-only mode reports itself as advisory rather than as a control', () => { const enforcer = new QueryBudgetEnforcer({ budget: DEFAULT_QUERY_BUDGET, store: new InMemoryWindowStore(), ledger: { record: () => undefined }, enforceInFlight: false, }); const described = enforcer.describeEnforcement(); expect(described.binding).toBe(false); expect(described.in_flight).toContain('NOT ENFORCED'); evidence.advisory_posture_is_declared = described; }); it('writes the evidence file', () => { fs.mkdirSync(path.dirname(EVIDENCE_PATH), { recursive: true }); fs.writeFileSync(EVIDENCE_PATH, `${JSON.stringify(evidence, null, 2)}\n`); expect(fs.existsSync(EVIDENCE_PATH)).toBe(true); }); });