260 lines
11 KiB
TypeScript
260 lines
11 KiB
TypeScript
/**
|
|
* tests/integration/query-cancellation-budget.integration.test.ts
|
|
* ================================================================
|
|
* BI-P019 (cancellation) and BI-P021 (cost/budget) against a live ClickHouse.
|
|
*
|
|
* Both prompts are only real if something ACTUALLY STOPS. So both are tested
|
|
* by starting a genuine CPU bomb and requiring the server to stop working:
|
|
*
|
|
* * the query must disappear from `system.processes`;
|
|
* * `system.query_log` must record it as ExceptionWhileProcessing with
|
|
* ClickHouse code 394 (QUERY_WAS_CANCELLED) — the server's own account of
|
|
* what happened, not ours;
|
|
* * and the COUNTERFACTUAL must hold: an identical bomb left alone for the
|
|
* same interval is still running. Without that, "the query was gone after
|
|
* we cancelled it" is equally consistent with "the query was short".
|
|
*
|
|
* BI-P021's in-flight enforcement gets the same treatment: a tenant window
|
|
* budget of one million rows, a query that scans far more, and the requirement
|
|
* that the enforcer cancels it MID-FLIGHT with `rows_read` already non-zero —
|
|
* i.e. it was running, it was observed running, and it was stopped.
|
|
*
|
|
* Gated on CLICKHOUSE_URL. With the variable set and the server down, this
|
|
* suite FAILS; it does not degrade to a skip.
|
|
*/
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { beforeAll, describe, expect, it } from 'vitest';
|
|
import { ClickHouseProcessControl } from '@/src/analytics/execution/clickhouse-process-control';
|
|
import { cancelAndVerify, cancelOrThrow } from '@/src/analytics/execution/cancellation';
|
|
import {
|
|
DEFAULT_QUERY_BUDGET,
|
|
InMemoryWindowStore,
|
|
QueryBudgetEnforcer,
|
|
} from '@/src/analytics/cost/query-budget';
|
|
|
|
const BASE_URL = process.env.CLICKHOUSE_URL ?? '';
|
|
const AUTH = process.env.CLICKHOUSE_AUTH ?? 'default:';
|
|
const suite = BASE_URL ? describe : describe.skip;
|
|
const EVIDENCE_PATH = path.join(process.cwd(), 'evidence/bi-p019-p021/cancellation-and-budget.json');
|
|
|
|
/** Big enough that it cannot finish inside any of the windows below. */
|
|
const BOMB = 'SELECT sum(number) AS s FROM numbers_mt(200000000000)';
|
|
|
|
interface Res {
|
|
ok: boolean;
|
|
status: number;
|
|
body: string;
|
|
}
|
|
|
|
async function send(sql: string, opts: { queryId?: string; timeoutMs?: number } = {}): Promise<Res> {
|
|
const url = new URL(BASE_URL);
|
|
url.searchParams.set('query', sql);
|
|
if (opts.queryId) url.searchParams.set('query_id', opts.queryId);
|
|
try {
|
|
const res = await fetch(url.toString(), {
|
|
method: 'POST',
|
|
headers: { Authorization: `Basic ${Buffer.from(AUTH).toString('base64')}` },
|
|
body: '',
|
|
signal: AbortSignal.timeout(opts.timeoutMs ?? 120_000),
|
|
});
|
|
return { ok: res.ok, status: res.status, body: await res.text() };
|
|
} catch (e) {
|
|
return { ok: false, status: 0, body: String(e) };
|
|
}
|
|
}
|
|
|
|
/** Fire and forget: the bomb runs on the server while the test carries on. */
|
|
function launch(queryId: string): Promise<Res> {
|
|
return send(BOMB, { queryId, timeoutMs: 120_000 });
|
|
}
|
|
|
|
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
|
|
const control = new ClickHouseProcessControl({ baseUrl: BASE_URL || 'http://unset', basicAuth: AUTH });
|
|
const evidence: Record<string, unknown> = {};
|
|
|
|
async function waitUntilRunning(queryId: string, timeoutMs = 10_000): Promise<boolean> {
|
|
const deadline = Date.now() + timeoutMs;
|
|
while (Date.now() < deadline) {
|
|
if (await control.isRunning(queryId)) return true;
|
|
await sleep(50);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/** ClickHouse's own verdict, from its query log. 394 = QUERY_WAS_CANCELLED. */
|
|
async function queryLogVerdict(
|
|
queryId: string,
|
|
): Promise<{ type: string; exception_code: number } | undefined> {
|
|
await send('SYSTEM FLUSH LOGS');
|
|
const res = await send(
|
|
`SELECT type, exception_code FROM system.query_log WHERE query_id = '${queryId}' ` +
|
|
`AND type != 'QueryStart' ORDER BY event_time_microseconds DESC LIMIT 1 FORMAT JSON`,
|
|
);
|
|
if (!res.ok) return undefined;
|
|
const row = (JSON.parse(res.body) as { data: Array<{ type: string; exception_code: number | string }> })
|
|
.data[0];
|
|
return row ? { type: row.type, exception_code: Number(row.exception_code) } : undefined;
|
|
}
|
|
|
|
beforeAll(async () => {
|
|
if (!BASE_URL) return;
|
|
const v = await send('SELECT version()');
|
|
if (!v.ok) throw new Error(`CLICKHOUSE_URL is set but ClickHouse is not answering: ${v.status}`);
|
|
evidence.clickhouse_version = v.body.trim();
|
|
evidence.recorded_at = new Date().toISOString();
|
|
}, 60_000);
|
|
|
|
suite('BI-P019 — cancellation stops an expensive query, and it is VERIFIED', () => {
|
|
it('counterfactual: an identical bomb left alone is still running after the same interval', async () => {
|
|
const queryId = `opt_cf_${Date.now()}`;
|
|
void launch(queryId);
|
|
expect(await waitUntilRunning(queryId)).toBe(true);
|
|
await sleep(2_000);
|
|
const stillRunning = await control.isRunning(queryId);
|
|
evidence.counterfactual_still_running_after_2s = stillRunning;
|
|
expect(stillRunning, 'the bomb finished on its own — it is not expensive enough to prove anything').toBe(
|
|
true,
|
|
);
|
|
// Clean up, and incidentally exercise the same path.
|
|
await cancelAndVerify(control, queryId, { timeoutMs: 10_000 });
|
|
}, 90_000);
|
|
|
|
it("cancellation is observed to stop it, and ClickHouse's own log agrees", async () => {
|
|
const queryId = `opt_cancel_${Date.now()}`;
|
|
void launch(queryId);
|
|
expect(await waitUntilRunning(queryId)).toBe(true);
|
|
// Let it accumulate real work so "it stopped" is a statement about a
|
|
// running query, not about one that had barely begun.
|
|
await sleep(1_000);
|
|
const progressBefore = await control.progress(queryId);
|
|
expect(progressBefore?.rows_read ?? 0).toBeGreaterThan(0);
|
|
|
|
const outcome = await cancelAndVerify(control, queryId, { timeoutMs: 10_000, pollIntervalMs: 100 });
|
|
expect(outcome.kill_accepted).toBe(true);
|
|
expect(outcome.verified_stopped, 'the query outlived its cancellation deadline').toBe(true);
|
|
expect(outcome.already_finished).toBe(false);
|
|
expect(outcome.elapsed_ms).toBeLessThan(10_000);
|
|
expect(await control.isRunning(queryId)).toBe(false);
|
|
|
|
const verdict = await queryLogVerdict(queryId);
|
|
evidence.cancellation = {
|
|
rows_read_before_cancel: progressBefore?.rows_read,
|
|
elapsed_ms_to_verified_stop: outcome.elapsed_ms,
|
|
verified_stopped: outcome.verified_stopped,
|
|
clickhouse_query_log_type: verdict?.type,
|
|
clickhouse_exception_code: verdict?.exception_code,
|
|
};
|
|
// 394 = QUERY_WAS_CANCELLED. The server saying it, not us.
|
|
expect(verdict?.exception_code, `query_log verdict was ${JSON.stringify(verdict)}`).toBe(394);
|
|
}, 90_000);
|
|
|
|
it('cancelling a query that already finished reports already_finished, not a false stop', async () => {
|
|
const queryId = `opt_done_${Date.now()}`;
|
|
const res = await send('SELECT 1', { queryId });
|
|
expect(res.ok).toBe(true);
|
|
const outcome = await cancelAndVerify(control, queryId, { timeoutMs: 2_000 });
|
|
expect(outcome.already_finished).toBe(true);
|
|
expect(outcome.verified_stopped).toBe(true);
|
|
});
|
|
|
|
it('cancelOrThrow raises when the query is not observed to stop in time', async () => {
|
|
const queryId = `opt_deadline_${Date.now()}`;
|
|
void launch(queryId);
|
|
expect(await waitUntilRunning(queryId)).toBe(true);
|
|
// A deadline shorter than ClickHouse can plausibly unwind in, with the
|
|
// kill request suppressed so the query genuinely keeps running. The
|
|
// point is that the module REPORTS a failure to stop rather than
|
|
// returning a cheerful boolean.
|
|
const noKill = {
|
|
engine: 'clickhouse',
|
|
isRunning: (id: string) => control.isRunning(id),
|
|
requestKill: async () => false,
|
|
progress: (id: string) => control.progress(id),
|
|
};
|
|
await expect(cancelOrThrow(noKill, queryId, { timeoutMs: 300, pollIntervalMs: 50 })).rejects.toThrow(
|
|
/not observed to stop/,
|
|
);
|
|
await cancelAndVerify(control, queryId, { timeoutMs: 10_000 });
|
|
}, 90_000);
|
|
});
|
|
|
|
suite('BI-P021 — the budget actually stops a running query', () => {
|
|
it('an in-flight window breach cancels the query mid-scan, with the cancellation verified', async () => {
|
|
const store = new InMemoryWindowStore();
|
|
const settled: unknown[] = [];
|
|
const enforcer = new QueryBudgetEnforcer({
|
|
budget: { ...DEFAULT_QUERY_BUDGET, maxRowsScanned: 1_000_000, maxCostUnits: 1e9 },
|
|
store,
|
|
ledger: { record: (e) => void settled.push(e) },
|
|
control,
|
|
enforceInFlight: true,
|
|
pollIntervalMs: 100,
|
|
storeIsShared: false,
|
|
});
|
|
|
|
const queryId = `opt_budget_${Date.now()}`;
|
|
void launch(queryId);
|
|
expect(await waitUntilRunning(queryId)).toBe(true);
|
|
|
|
const outcome = await enforcer.enforceInFlight('t_alpha', queryId);
|
|
expect(outcome.terminated, 'the budget watched the query run past its window and did nothing').toBe(true);
|
|
expect(outcome.reason).toBe('rows');
|
|
// It was RUNNING when it was stopped — not stopped before it started.
|
|
expect(outcome.observed.rows_read).toBeGreaterThan(1_000_000);
|
|
expect(outcome.cancellation?.verified_stopped).toBe(true);
|
|
expect(await control.isRunning(queryId)).toBe(false);
|
|
|
|
const verdict = await queryLogVerdict(queryId);
|
|
expect(verdict?.exception_code).toBe(394);
|
|
|
|
await enforcer.settle({
|
|
tenant_id: 't_alpha',
|
|
principal_id: 'u_1',
|
|
query_id: queryId,
|
|
cost: {
|
|
rows_scanned: outcome.observed.rows_read,
|
|
bytes_scanned: outcome.observed.bytes_read,
|
|
execution_ms: outcome.observed.elapsed_ms,
|
|
engine: 'clickhouse',
|
|
},
|
|
terminated_by_budget: true,
|
|
});
|
|
expect(settled).toHaveLength(1);
|
|
|
|
evidence.budget_in_flight = {
|
|
budget_max_rows_scanned: 1_000_000,
|
|
observed_rows_read_at_cancel: outcome.observed.rows_read,
|
|
observed_elapsed_ms_at_cancel: outcome.observed.elapsed_ms,
|
|
polls_before_breach: outcome.polls,
|
|
overshoot_factor: Math.round(outcome.observed.rows_read / 1_000_000),
|
|
overshoot_note:
|
|
'in-flight enforcement is granular to the poll interval; the query overshot the window ' +
|
|
'by this factor before the poll that caught it. A bound, not a hard cap.',
|
|
cancellation_verified: outcome.cancellation?.verified_stopped,
|
|
clickhouse_exception_code: verdict?.exception_code,
|
|
charged_for_work_done_before_cancel: true,
|
|
};
|
|
}, 120_000);
|
|
|
|
it('admission-only mode reports itself as advisory rather than as a control', () => {
|
|
const enforcer = new QueryBudgetEnforcer({
|
|
budget: DEFAULT_QUERY_BUDGET,
|
|
store: new InMemoryWindowStore(),
|
|
ledger: { record: () => undefined },
|
|
enforceInFlight: false,
|
|
});
|
|
const described = enforcer.describeEnforcement();
|
|
expect(described.binding).toBe(false);
|
|
expect(described.in_flight).toContain('NOT ENFORCED');
|
|
evidence.advisory_posture_is_declared = described;
|
|
});
|
|
|
|
it('writes the evidence file', () => {
|
|
fs.mkdirSync(path.dirname(EVIDENCE_PATH), { recursive: true });
|
|
fs.writeFileSync(EVIDENCE_PATH, `${JSON.stringify(evidence, null, 2)}\n`);
|
|
expect(fs.existsSync(EVIDENCE_PATH)).toBe(true);
|
|
});
|
|
});
|