fix(recovery): restore source removed by production documentation merge

This commit is contained in:
2026-11-10 03:42:10 +01:00
parent 585bdd4b44
commit 3d2fc6f913
957 changed files with 115754 additions and 0 deletions
+183
View File
@@ -0,0 +1,183 @@
/**
* src/analytics/execution/cancellation.ts
* ========================================
* BI-P019 — QUERY CANCELLATION.
*
* "Integrate engine cancellation. DoD: User cancellation stops expensive
* query within defined timeout."
*
* THE THING THAT MAKES THIS REAL: "stops" is VERIFIED, not requested.
*
* The obvious implementation aborts the HTTP request and issues a KILL. Both
* of those are requests. The client abort stops US waiting; it does not stop
* the server, which keeps burning CPU on a query nobody will read. `KILL QUERY`
* without `SYNC` returns immediately and tells you nothing. Even `KILL QUERY
* SYNC` can return while the query is still winding down.
*
* So `cancelAndVerify` does three things and reports on all three:
* 1. abort the client side;
* 2. issue the engine's kill;
* 3. POLL the engine's process list until the query is GONE, or the deadline
* passes — and if the deadline passes, it says so with
* `verified_stopped: false` rather than returning a cheerful `true`.
*
* A caller that treats `cancelQuery()` returning `true` as "the query stopped"
* is the failure this module exists to prevent, which is why the return type
* is an outcome object and not a boolean.
*/
import { SiaxError } from '../../platform/errors';
/** Whatever can list and kill running queries. ClickHouse and Trino both can. */
export interface EngineProcessControl {
readonly engine: string;
/** True when a query with this id is still running on the engine. */
isRunning(queryId: string): Promise<boolean>;
/** Ask the engine to stop it. Returning true means "accepted", not "stopped". */
requestKill(queryId: string): Promise<boolean>;
/** Live progress for the running query, when the engine exposes it (BI-P021). */
progress?(queryId: string): Promise<QueryProgress | undefined>;
}
export interface QueryProgress {
rows_read: number;
bytes_read: number;
elapsed_ms: number;
}
export interface CancellationOutcome {
query_id: string;
requested_at: string;
/** The engine accepted the kill request. */
kill_accepted: boolean;
/**
* The query was OBSERVED to be gone from the engine's process list.
* `false` means it may still be running — the honest answer.
*/
verified_stopped: boolean;
/** Milliseconds from the cancel request to the observation. */
elapsed_ms: number;
/** The deadline this cancellation was held to. */
timeout_ms: number;
/** Present when the query had already finished before the request landed. */
already_finished: boolean;
}
export interface CancellationOptions {
/** The defined timeout BI-P019's DoD refers to. */
timeoutMs?: number;
pollIntervalMs?: number;
/** Aborts the client-side wait. */
abort?: () => void;
now?: () => number;
sleep?: (ms: number) => Promise<void>;
}
export const DEFAULT_CANCELLATION_TIMEOUT_MS = 5_000;
const DEFAULT_POLL_INTERVAL_MS = 100;
const defaultSleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
export async function cancelAndVerify(
control: EngineProcessControl,
queryId: string,
opts: CancellationOptions = {},
): Promise<CancellationOutcome> {
if (!queryId?.trim()) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'cancellation requires a query id');
}
const timeoutMs = opts.timeoutMs ?? DEFAULT_CANCELLATION_TIMEOUT_MS;
const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
const now = opts.now ?? Date.now;
const sleep = opts.sleep ?? defaultSleep;
const startedMs = now();
const requestedAt = new Date(startedMs).toISOString();
// Was it even running? Cancelling a finished query is a no-op, and reporting
// it as a successful stop would overstate what happened.
//
// FAIL CLOSED when the engine cannot be asked: an unreachable engine means
// "unknown", and the safe reading of unknown is "still running" — so the
// kill is still issued and the verification loop still has to observe the
// query gone. Letting this throw would abandon the cancellation entirely at
// the exact moment the operator most needs it to be attempted.
let runningBefore: boolean;
try {
runningBefore = await control.isRunning(queryId);
} catch {
runningBefore = true;
}
// 1. client side: stop waiting.
opts.abort?.();
// 2. engine side: ask it to stop.
let killAccepted = false;
try {
killAccepted = await control.requestKill(queryId);
} catch {
killAccepted = false;
}
if (!runningBefore) {
return {
query_id: queryId,
requested_at: requestedAt,
kill_accepted: killAccepted,
verified_stopped: true,
elapsed_ms: now() - startedMs,
timeout_ms: timeoutMs,
already_finished: true,
};
}
// 3. VERIFY. This loop is the whole point of the module.
let verified = false;
for (;;) {
let stillRunning: boolean;
try {
stillRunning = await control.isRunning(queryId);
} catch {
// An engine we cannot ask is an engine we cannot vouch for. Keep trying
// until the deadline, then report unverified.
stillRunning = true;
}
if (!stillRunning) {
verified = true;
break;
}
if (now() - startedMs >= timeoutMs) break;
await sleep(pollIntervalMs);
}
return {
query_id: queryId,
requested_at: requestedAt,
kill_accepted: killAccepted,
verified_stopped: verified,
elapsed_ms: now() - startedMs,
timeout_ms: timeoutMs,
already_finished: false,
};
}
/**
* The same call, for a caller that wants a failure rather than a report.
* Throws `RETRYABLE_FAILURE` when the query outlived the deadline — because
* "we asked it to stop and it did not" is an operational condition someone
* has to see, not a value to ignore.
*/
export async function cancelOrThrow(
control: EngineProcessControl,
queryId: string,
opts: CancellationOptions = {},
): Promise<CancellationOutcome> {
const outcome = await cancelAndVerify(control, queryId, opts);
if (!outcome.verified_stopped) {
throw new SiaxError(
'RETRYABLE_FAILURE',
`0pt.analytics.${control.engine}`,
`query was not observed to stop within ${outcome.timeout_ms}ms of cancellation`,
);
}
return outcome;
}
@@ -0,0 +1,138 @@
/**
* src/analytics/execution/clickhouse-process-control.ts
* ======================================================
* BI-P019/BI-P021 — the ClickHouse implementation of `EngineProcessControl`.
*
* Three statements, each chosen for a measured reason:
*
* isRunning `SELECT count() FROM system.processes WHERE query_id = …`
* — the authority on whether the server is still working. Note
* that `KILL QUERY SYNC` returning is NOT this: a killed query
* can still appear here briefly while it unwinds, which is
* exactly why cancellation polls instead of trusting the kill.
*
* requestKill `KILL QUERY WHERE query_id = … ASYNC` — ASYNC, not SYNC,
* because the verification loop is what establishes the stop and
* a SYNC kill would block this call for up to the query's
* remaining lifetime while providing no stronger guarantee.
*
* progress `read_rows`, `read_bytes` and `elapsed` from the same table —
* live counters, which is what makes BI-P021's in-flight budget
* enforcement possible at all.
*
* KILL and the system-table reads are ADMINISTRATIVE. They run on the engine's
* own credential, never on the tenant's restricted user (which has no access
* to system.processes — measured: ACCESS_DENIED, code 497). The tenant-scoping
* check therefore happens HERE, on the query id, before any statement is sent.
*/
import { SiaxError } from '../../platform/errors';
import type { EngineProcessControl, QueryProgress } from './cancellation';
export interface ClickHouseProcessControlOptions {
baseUrl: string;
/** "user:password" for the ADMIN credential. From a secret ref. */
basicAuth?: string;
timeoutMs?: number;
/**
* Restricts every operation to query ids carrying this prefix, so a caller
* cannot kill or inspect another tenant's work through this object.
*/
queryIdPrefix?: string;
}
export class ClickHouseProcessControl implements EngineProcessControl {
readonly engine = 'clickhouse';
private readonly baseUrl: string;
private readonly authHeader?: string;
private readonly timeoutMs: number;
private readonly prefix?: string;
constructor(opts: ClickHouseProcessControlOptions) {
this.baseUrl = opts.baseUrl.replace(/\/+$/, '');
this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString('base64')}` : undefined;
this.timeoutMs = opts.timeoutMs ?? 10_000;
this.prefix = opts.queryIdPrefix;
}
private assertScoped(queryId: string): void {
if (this.prefix && !queryId.startsWith(this.prefix)) {
throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics', 'query does not belong to this scope');
}
if (!/^[A-Za-z0-9_-]{1,128}$/.test(queryId)) {
// The id reaches ClickHouse as a bound parameter, but a malformed id is
// a bug worth surfacing rather than a string worth binding.
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'malformed query id');
}
}
async isRunning(queryId: string): Promise<boolean> {
this.assertScoped(queryId);
const body = await this.send(
'SELECT count() AS n FROM system.processes WHERE query_id = {qid:String} FORMAT JSON',
{ param_qid: queryId },
);
const n = Number((JSON.parse(body) as { data: Array<{ n: string | number }> }).data[0]?.n ?? 0);
return n > 0;
}
async requestKill(queryId: string): Promise<boolean> {
this.assertScoped(queryId);
await this.send(
'KILL QUERY WHERE query_id = {qid:String} ASYNC FORMAT JSON',
{ param_qid: queryId },
false,
);
return true;
}
async progress(queryId: string): Promise<QueryProgress | undefined> {
this.assertScoped(queryId);
const body = await this.send(
'SELECT read_rows, read_bytes, toUInt64(elapsed * 1000) AS elapsed_ms FROM system.processes ' +
'WHERE query_id = {qid:String} FORMAT JSON',
{ param_qid: queryId },
);
const row = (JSON.parse(body) as { data: Array<Record<string, string | number>> }).data[0];
if (!row) return undefined;
return {
rows_read: Number(row.read_rows ?? 0),
bytes_read: Number(row.read_bytes ?? 0),
elapsed_ms: Number(row.elapsed_ms ?? 0),
};
}
private async send(sql: string, params: Record<string, string>, expectJson = true): Promise<string> {
const url = new URL(this.baseUrl);
url.searchParams.set('query', sql);
for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
// NOT readonly: KILL mutates server state. The other two would tolerate
// readonly, but sharing one path keeps the credential story simple.
url.searchParams.set('max_execution_time', '10');
const headers: Record<string, string> = {};
if (this.authHeader) headers.Authorization = this.authHeader;
let res: Response;
try {
res = await fetch(url.toString(), {
method: 'POST',
headers,
body: '',
signal: AbortSignal.timeout(this.timeoutMs),
});
} catch (cause) {
throw new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'clickhouse unreachable', { cause });
}
const text = await res.text();
if (!res.ok) {
const code = /Code:\s*(\d+)/.exec(text)?.[1] ?? String(res.status);
throw new SiaxError(
'PERMANENT_FAILURE',
'clickhouse',
`process-control statement failed (CH code ${code})`,
);
}
if (expectJson && !text.trim()) {
throw new SiaxError('PERMANENT_FAILURE', 'clickhouse', 'empty response from process-control statement');
}
return text;
}
}
@@ -0,0 +1,274 @@
/**
* src/analytics/execution/query-execution-record.ts
* ==================================================
* BI-P018 — QUERY EXECUTION MODEL.
*
* Every field BI-P018 lists, verbatim, and one it lists by its absence:
*
* "Every production query has traceable execution metadata WITHOUT
* unnecessarily storing sensitive raw SQL."
*
* So the record carries `compiled_query_hash` and never the statement. That is
* enforced, not documented: `assertNoRawSql` scans a finished record for the
* statement it describes, and
* tests/analytics/query-execution-model.test.ts fails if any field carries it.
*
* The ONE place raw SQL is retained is the analyst statement history
* (BI-P017), where a privileged human ran unreviewed SQL against tenant data
* and "who ran what" has to survive. That history is a separate record with a
* separate retention posture; this one is the per-execution telemetry that
* every query in the estate produces, and it must be safe to keep for a long
* time and to ship to AUD0.
*
* BI-P002's `QueryExecution` entity is the persisted row; this module is the
* lifecycle that produces it, plus the four fields BI-P018 adds to it
* (compiled_query_hash, datasets, metrics, cache_status, cost_units).
*/
import crypto from 'node:crypto';
import { SiaxError } from '../../platform/errors';
import type { ExecutionState } from '../domain/entities';
/** BI-P020's outcome, recorded per execution so cache behaviour is measurable. */
export const CACHE_STATUSES = ['hit', 'miss', 'bypass', 'stale_refused', 'not_cacheable'] as const;
export type CacheStatus = (typeof CACHE_STATUSES)[number];
/** Which surface produced the query — the "user/workload" axis BI-P018 names. */
export const WORKLOAD_KINDS = [
'dashboard',
'report',
'api',
'analyst_sql',
'ai_analyst',
'scheduled',
'internal',
] as const;
export type WorkloadKind = (typeof WORKLOAD_KINDS)[number];
export interface QueryExecutionRecord {
query_id: string;
execution_id: string;
tenant_id: string;
project_id?: string;
/** The CL0UD principal. A human, a service, or a job — never absent. */
principal_id: string;
workload: WorkloadKind;
submitted_at: string;
started_at?: string;
completed_at?: string;
engine: string;
/**
* sha256 of the COMPILED statement plus its bound parameter values.
*
* Parameters are included deliberately: two executions of the same SQL with
* different filters are different work with different cost, and a hash that
* collapsed them would make the cost record meaningless. They are HASHED,
* not stored, so the record still carries no tenant data.
*/
compiled_query_hash: string;
datasets: string[];
metrics: string[];
status: ExecutionState;
/** Rows RETURNED to the caller. */
rows?: number;
/** Rows and bytes SCANNED — the cost axis, not the result size. */
rows_scanned?: number;
bytes_scanned?: number;
duration_ms?: number;
cache_status: CacheStatus;
/** BI-P021 units. Present once the query has settled, including on failure. */
cost_units?: number;
/** A SiaxErrorCode. Never an engine body, never a fragment of the statement. */
error?: string;
correlation_id?: string;
}
/**
* Hash a compiled statement and its bound parameters into the one value the
* record keeps.
*
* Parameters are sorted so two logically identical executions hash equal
* regardless of key iteration order — otherwise the cache-key and cost
* aggregations built on this hash would fragment for no reason.
*/
export function compiledQueryHash(sql: string, params: Record<string, string> = {}): string {
const canonical = JSON.stringify([
sql.trim(),
Object.keys(params)
.sort()
.map((k) => [k, params[k]]),
]);
return crypto.createHash('sha256').update(canonical).digest('hex');
}
/**
* The legal state machine. Written down because "status" fields drift into
* meaninglessness when any value can follow any other — a `succeeded` row that
* later becomes `running` makes every downstream aggregation a lie.
*/
const LEGAL_TRANSITIONS: Record<ExecutionState, readonly ExecutionState[]> = {
queued: ['running', 'cancelled', 'rejected', 'failed'],
running: ['succeeded', 'failed', 'cancelled'],
succeeded: [],
failed: [],
cancelled: [],
rejected: [],
};
export function assertLegalTransition(from: ExecutionState, to: ExecutionState): void {
if (!LEGAL_TRANSITIONS[from].includes(to)) {
throw new SiaxError('CONFLICT', '0pt.analytics', `illegal execution transition ${from} -> ${to}`);
}
}
/** Terminal states: a record in one of these will never change again. */
export const TERMINAL_STATES: readonly ExecutionState[] = ['succeeded', 'failed', 'cancelled', 'rejected'];
export interface ExecutionSink {
write(record: QueryExecutionRecord): Promise<void> | void;
}
export interface ExecutionRecorderOptions {
tenant_id: string;
project_id?: string;
principal_id: string;
workload: WorkloadKind;
engine: string;
correlation_id?: string;
sink: ExecutionSink;
now?: () => number;
}
/**
* Owns one execution's record from submission to a terminal state.
*
* Every transition WRITES. A recorder that only writes at the end loses
* exactly the executions that matter: the ones that never finished.
*/
export class QueryExecutionRecorder {
private record: QueryExecutionRecord;
private readonly sink: ExecutionSink;
private readonly now: () => number;
constructor(
opts: ExecutionRecorderOptions,
queryId: string,
compiled: { hash: string; datasets: string[]; metrics: string[] },
) {
if (!opts.tenant_id?.trim()) {
throw new SiaxError(
'UNAUTHENTICATED',
'0pt.analytics',
'an execution record requires a verified tenant',
);
}
if (!opts.principal_id?.trim()) {
throw new SiaxError(
'UNAUTHENTICATED',
'0pt.analytics',
'an execution record requires a verified principal',
);
}
this.now = opts.now ?? Date.now;
this.sink = opts.sink;
this.record = {
query_id: queryId,
execution_id: crypto.randomUUID(),
tenant_id: opts.tenant_id,
project_id: opts.project_id,
principal_id: opts.principal_id,
workload: opts.workload,
submitted_at: new Date(this.now()).toISOString(),
engine: opts.engine,
compiled_query_hash: compiled.hash,
datasets: [...compiled.datasets],
metrics: [...compiled.metrics],
status: 'queued',
cache_status: 'miss',
correlation_id: opts.correlation_id,
};
}
get current(): QueryExecutionRecord {
return { ...this.record, datasets: [...this.record.datasets], metrics: [...this.record.metrics] };
}
async submitted(): Promise<void> {
await this.sink.write(this.current);
}
async started(): Promise<void> {
assertLegalTransition(this.record.status, 'running');
this.record.status = 'running';
this.record.started_at = new Date(this.now()).toISOString();
await this.sink.write(this.current);
}
/** A cache hit never reached the engine; record it as such, not as a fast query. */
async servedFromCache(rows: number, costUnits = 0): Promise<void> {
assertLegalTransition(this.record.status, 'running');
this.record.status = 'running';
this.record.started_at = this.record.started_at ?? new Date(this.now()).toISOString();
this.record.cache_status = 'hit';
await this.finish('succeeded', { rows, rows_scanned: 0, bytes_scanned: 0, cost_units: costUnits });
}
setCacheStatus(status: CacheStatus): void {
this.record.cache_status = status;
}
async finish(
status: ExecutionState,
stats: {
rows?: number;
rows_scanned?: number;
bytes_scanned?: number;
cost_units?: number;
error?: string;
},
): Promise<QueryExecutionRecord> {
assertLegalTransition(this.record.status, status);
const completedMs = this.now();
this.record.status = status;
this.record.completed_at = new Date(completedMs).toISOString();
const startedMs = this.record.started_at
? Date.parse(this.record.started_at)
: Date.parse(this.record.submitted_at);
this.record.duration_ms = Math.max(0, completedMs - startedMs);
if (stats.rows !== undefined) this.record.rows = stats.rows;
if (stats.rows_scanned !== undefined) this.record.rows_scanned = stats.rows_scanned;
if (stats.bytes_scanned !== undefined) this.record.bytes_scanned = stats.bytes_scanned;
if (stats.cost_units !== undefined) this.record.cost_units = stats.cost_units;
if (stats.error !== undefined) this.record.error = stats.error;
assertNoRawSql(this.record);
await this.sink.write(this.current);
return this.current;
}
}
/**
* Refuse to emit a record that carries statement text.
*
* Checked at runtime rather than trusted to the type, because the fields that
* leak are the string ones a future change widens — `error` becoming an engine
* body is the obvious one, and ClickHouse's error text echoes the query.
*/
export function assertNoRawSql(record: QueryExecutionRecord): void {
const suspicious = /\b(select|insert|update|delete|drop|create|alter|from|where|join|union)\b/i;
const stringFields: Array<[string, string | undefined]> = [
['error', record.error],
['query_id', record.query_id],
['compiled_query_hash', record.compiled_query_hash],
['correlation_id', record.correlation_id],
...record.datasets.map((d, i) => [`datasets[${i}]`, d] as [string, string]),
...record.metrics.map((m, i) => [`metrics[${i}]`, m] as [string, string]),
];
for (const [name, value] of stringFields) {
if (value && suspicious.test(value)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics',
`execution record field "${name}" looks like SQL; BI-P018 stores a hash, never the statement`,
);
}
}
}