Files
0pt/src/analytics/sql/statement-guard.ts
T

517 lines
18 KiB
TypeScript

/**
* src/analytics/sql/statement-guard.ts
* =====================================
* BI-P017 — ANALYST SQL CONTRACT, layer 2 (statement admission).
*
* READ THIS BEFORE TRUSTING IT.
*
* This module is a GUARD, not a proof. It is the second of four layers, and
* it is the WEAKEST of the four, because it reasons about SQL text without
* being ClickHouse's parser. A guard that reads a dialect it does not
* implement can always be wrong about some string. The layers below it are
* the ones that hold when it is:
*
* 1. ROLE — only a CL0UD-verified analyst role reaches a session
* at all (analyst-sql-session.ts).
* 2. THIS GUARD — statement admission: one read-only statement, no
* banned construct, every table reference resolved
* into the tenant's own catalog.
* 3. GRANTS — the session authenticates as a per-tenant ClickHouse
* user with `readonly = 1` and SELECT granted ONLY on
* that tenant's catalog. A statement this guard wrongly
* admits still cannot read a table the user cannot see.
* 4. VIEW PREDICATE — the tenant catalog contains VIEWS, never base tables,
* and each view bakes `WHERE tenant_id = '<tenant>'`
* into its own definition. A row policy on the base
* tables repeats it server-side.
*
* tests/integration/analyst-sql-clickhouse.integration.test.ts runs the
* adversarial corpus against a live ClickHouse with layer 2 REMOVED, and
* requires layers 3 and 4 to refuse it anyway. A defence-in-depth claim that
* has never been tested with a layer removed is not evidence — this estate
* has shipped several.
*
* DESIGN NOTES that are load-bearing:
*
* * Identifiers must resolve to an ALLOW-LIST. The deny-lists below are a
* second opinion and a better error message, never the mechanism. Every
* table reference is checked positively against the tenant's catalog.
* * Outside string literals the statement must be ASCII. Homoglyph and
* zero-width tricks against keyword matching stop being expressible.
* * The FROM/JOIN scan is CONSERVATIVE where it is unsure, and that costs
* something worth naming: ClickHouse's `ARRAY JOIN arr` is a legitimate
* construct whose target is a COLUMN, not a table, and this guard refuses
* it because the token after JOIN is neither a CTE nor a qualified object.
* A false refusal an analyst can see and report is the right side of that
* trade; a false ADMISSION is not. Supporting it means teaching the scan
* which JOIN flavours take a non-table target, which is the direction that
* grows holes, so it stays refused until someone needs it and can add it
* WITH a corpus entry proving the narrowing did not open anything else.
* * Comments and string literals are removed BEFORE any keyword scan, using
* ClickHouse's own lexical rules (`--` and `#` to end of line, `/* *\/`
* non-nesting, `'` with backslash and doubled-quote escapes, `"` and
* backtick as identifier quotes).
*/
import { SiaxError } from '../../platform/errors';
export interface StatementGuardOptions {
/**
* The ONLY database name a table reference may carry. This is the tenant's
* own catalog of pre-filtered views, not the shared analytics database.
*/
allowedCatalog: string;
/** Object names inside that catalog the analyst may read. */
allowedObjects: readonly string[];
/** Hard ceiling on statement length; a megabyte of SQL is an attack, not a query. */
maxLength?: number;
}
export interface GuardedStatement {
/** The statement as submitted, unchanged. */
raw: string;
/** Comments and literals removed — what the keyword scan actually saw. */
stripped: string;
/** Literals replaced by `?`, for statement history that is not a PII dump. */
normalized: string;
/** Fully-qualified objects the statement reads, resolved against the catalog. */
referencedObjects: string[];
/** Common-table-expression names defined by the statement itself. */
cteNames: string[];
}
const DEFAULT_MAX_LENGTH = 20_000;
/**
* Statement-level constructs that make a statement not-a-read. Matched as
* whole words against the stripped text.
*
* `SETTINGS` is here because a trailing `SETTINGS max_rows_to_read = …` is the
* obvious way to try to lift the governance ceiling from inside the query.
* `FORMAT` is here because the session appends its own; a caller-supplied one
* is either a duplicate or an attempt at `FORMAT ...` side effects.
*/
const BANNED_KEYWORDS = [
'insert',
'update',
'delete',
'drop',
'alter',
'create',
'truncate',
'rename',
'attach',
'detach',
'replace',
'optimize',
'grant',
'revoke',
'kill',
'system',
'use',
'set',
'settings',
'exchange',
'move',
'undrop',
'backup',
'restore',
'outfile',
'infile',
'format',
'watch',
'check',
'explain',
'show',
'exists',
'call',
'execute',
'declare',
'begin',
'commit',
'rollback',
] as const;
/**
* Functions that read something other than a table the grants control:
* the filesystem, the network, another server, a dictionary, an executable.
* ClickHouse table functions are the whole reason an analyst SQL surface is
* dangerous — `SELECT * FROM url('http://…')` needs no table privilege at all.
*/
const BANNED_FUNCTIONS = [
'file',
'url',
'urlcluster',
's3',
's3cluster',
'gcs',
'azureblobstorage',
'remote',
'remotesecure',
'cluster',
'clusterallreplicas',
'hdfs',
'hdfscluster',
'mysql',
'postgresql',
'mongodb',
'jdbc',
'odbc',
'sqlite',
'redis',
'deltalake',
'hudi',
'iceberg',
'executable',
'input',
'generaterandom',
'numbers_mt',
'merge',
'view',
'viewifpermitted',
'loop',
'fuzzjson',
'fuzzquery',
'dictget',
'dictgetordefault',
'dictgetornull',
'dictgetstring',
'dictgetuint64',
'dictgethierarchy',
'dictisin',
'dicthas',
'dictgetchildren',
'dictgetdescendants',
'filesystemavailable',
'filesystemfree',
'filesystemcapacity',
'currentdatabase',
'currentuser',
'hostname',
'fqdn',
'uptime',
'version',
'getsetting',
'getmacro',
'shardnum',
'shardcount',
'buildid',
'servertimezone',
'readwkt',
'structurefromjson',
's3queue',
'timeseries',
'mergetreeindex',
] as const;
/**
* Constructs that a whole-word keyword ban gets WRONG, so they get a pattern.
*
* `DESC` is the case that proves the point: banning the word breaks every
* `ORDER BY x DESC` an analyst will ever write, and the test corpus caught it
* immediately. What is actually dangerous is `DESCRIBE`/`DESC` used as a
* STATEMENT — including inside a subquery, where ClickHouse accepts
* `SELECT * FROM (DESCRIBE TABLE x)` and the FROM-scan sees only a `(`.
*/
const BANNED_PATTERNS: Array<{ re: RegExp; reason: string }> = [
{ re: /\bdesc(ribe)?\s+(table|\()/i, reason: 'DESCRIBE is not permitted in a read-only analyst session' },
{ re: /\bcurrent_user\b/i, reason: 'CURRENT_USER is not permitted in an analyst session' },
{ re: /\binto\s+outfile\b/i, reason: 'INTO OUTFILE is not permitted in an analyst session' },
];
/** Names that are catalogs an analyst may never reach, whatever the grants say. */
const BANNED_CATALOGS = [
'system',
'information_schema',
'INFORMATION_SCHEMA',
'default',
'opt_analytics',
'opt_scraped',
];
const IDENT = '[A-Za-z_][A-Za-z0-9_]*';
const QUALIFIED_RE = new RegExp(`^${IDENT}\\.${IDENT}$`);
function deny(detail: string): never {
throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics.sql', detail);
}
function invalid(detail: string): never {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.sql', detail);
}
interface StripResult {
stripped: string;
normalized: string;
/** Identifier tokens that were quoted, kept so they can still be resolved. */
hadQuotedIdentifier: boolean;
}
/**
* Remove comments and string literals using ClickHouse's lexical rules.
*
* Two outputs, deliberately: `stripped` keeps the literal DELIMITERS so the
* token stream still parses (`'x'` becomes `''`), and `normalized` replaces
* each literal with `?` so a statement history is searchable without being a
* copy of the tenant's data.
*
* An unterminated literal or comment is a REJECTION. Silently treating the
* rest of the statement as a string is exactly how a guard and an engine come
* to disagree about where the query ends.
*/
export function stripSqlText(sql: string): StripResult {
let stripped = '';
let normalized = '';
let hadQuotedIdentifier = false;
let i = 0;
const n = sql.length;
while (i < n) {
const c = sql[i];
const next = sql[i + 1];
// -- line comment
if (c === '-' && next === '-') {
while (i < n && sql[i] !== '\n') i += 1;
stripped += ' ';
normalized += ' ';
continue;
}
// # line comment (ClickHouse accepts `#` and `#!`)
if (c === '#') {
while (i < n && sql[i] !== '\n') i += 1;
stripped += ' ';
normalized += ' ';
continue;
}
// /* block comment */ — ClickHouse does NOT nest these
if (c === '/' && next === '*') {
const end = sql.indexOf('*/', i + 2);
if (end === -1) invalid('unterminated block comment');
i = end + 2;
stripped += ' ';
normalized += ' ';
continue;
}
// 'string literal'
if (c === "'") {
i += 1;
while (i < n) {
if (sql[i] === '\\') {
i += 2;
continue;
}
if (sql[i] === "'") {
if (sql[i + 1] === "'") {
i += 2;
continue;
}
break;
}
i += 1;
}
if (i >= n) invalid('unterminated string literal');
i += 1; // closing quote
stripped += "''";
normalized += '?';
continue;
}
// "quoted identifier" / `quoted identifier`
if (c === '"' || c === '`') {
const quote = c;
let ident = '';
i += 1;
while (i < n) {
if (sql[i] === '\\') {
ident += sql[i + 1] ?? '';
i += 2;
continue;
}
if (sql[i] === quote) {
if (sql[i + 1] === quote) {
ident += quote;
i += 2;
continue;
}
break;
}
ident += sql[i];
i += 1;
}
if (i >= n) invalid('unterminated quoted identifier');
i += 1;
hadQuotedIdentifier = true;
// The identifier is put back UNQUOTED so the table-reference scan sees
// it exactly as it would see a bare one. A quoted identifier that does
// not survive that scan is refused like any other.
if (!new RegExp(`^${IDENT}$`).test(ident)) {
deny(`quoted identifier "${ident.slice(0, 32)}" is not a plain name`);
}
stripped += ident;
normalized += ident;
continue;
}
stripped += c;
normalized += c;
i += 1;
}
return { stripped, normalized, hadQuotedIdentifier };
}
/**
* Admit a statement, or refuse it.
*
* Returns the parsed view of what was admitted; the caller (the session) is
* responsible for actually running it under the restricted user.
*/
export function guardAnalystStatement(sql: string, opts: StatementGuardOptions): GuardedStatement {
const maxLength = opts.maxLength ?? DEFAULT_MAX_LENGTH;
if (typeof sql !== 'string') invalid('statement must be a string');
const raw = sql.trim();
if (!raw) invalid('statement is empty');
if (raw.length > maxLength) invalid(`statement exceeds ${maxLength} characters`);
if (!opts.allowedCatalog || !new RegExp(`^${IDENT}$`).test(opts.allowedCatalog)) {
// A misconfigured session must not degrade to "any catalog".
deny('session has no valid tenant catalog');
}
if (BANNED_CATALOGS.includes(opts.allowedCatalog)) {
deny(`session catalog "${opts.allowedCatalog}" is a reserved catalog`);
}
if (opts.allowedObjects.length === 0) {
deny('session exposes no readable objects');
}
const { stripped, normalized } = stripSqlText(raw);
// ── ASCII-only outside literals ────────────────────────────────────────
// Homoglyphs, zero-width joiners and RTL overrides stop being expressible
// against the keyword scan. Values are unaffected: they live in literals,
// which have already been removed at this point.
// Written as a code-point scan rather than a character class: a regex of
// control characters is unreadable, trips `no-control-regex`, and is the
// kind of expression a reviewer nods at without checking. Permitted: tab,
// LF, CR and printable ASCII. Everything else, NUL included, is refused.
for (let i = 0; i < stripped.length; i += 1) {
const code = stripped.charCodeAt(i);
const printable = code >= 0x20 && code <= 0x7e;
const allowedWhitespace = code === 0x09 || code === 0x0a || code === 0x0d;
if (!printable && !allowedWhitespace) {
deny(
`statement contains a disallowed character (U+${code.toString(16).toUpperCase().padStart(4, '0')}) ` +
`outside a string literal`,
);
}
}
// ── exactly one statement ──────────────────────────────────────────────
const withoutTrailingSemi = stripped.replace(/;\s*$/, '');
if (withoutTrailingSemi.includes(';')) {
deny('multiple statements are not allowed in an analyst SQL session');
}
const lower = ` ${withoutTrailingSemi.toLowerCase().replace(/\s+/g, ' ').trim()} `;
// ── must be a read ─────────────────────────────────────────────────────
if (!/^\s*(select|with)\b/i.test(withoutTrailingSemi.trim())) {
deny('analyst SQL sessions accept SELECT and WITH statements only');
}
for (const kw of BANNED_KEYWORDS) {
if (new RegExp(`(^|[^A-Za-z0-9_])${kw}([^A-Za-z0-9_]|$)`).test(lower)) {
deny(`"${kw.toUpperCase()}" is not permitted in a read-only analyst session`);
}
}
for (const { re, reason } of BANNED_PATTERNS) {
if (re.test(lower)) deny(reason);
}
// ── banned functions, matched as `name(` ───────────────────────────────
const calls = [...lower.matchAll(new RegExp(`([a-z_][a-z0-9_]*)\\s*\\(`, 'g'))].map((m) => m[1]);
for (const call of calls) {
if ((BANNED_FUNCTIONS as readonly string[]).includes(call)) {
deny(`function "${call}" is not permitted in an analyst session`);
}
}
// ── table references: positively resolved against the catalog ──────────
const cteNames = collectCteNames(withoutTrailingSemi);
const referencedObjects = collectTableReferences(withoutTrailingSemi, cteNames, opts);
if (referencedObjects.length === 0) {
// A SELECT with no FROM (`SELECT 1`) is harmless, but so is refusing it,
// and "no table reference" is also what a successful parser evasion looks
// like. Refuse: an analyst session exists to read tables.
deny('statement references no readable object in the tenant catalog');
}
return { raw, stripped, normalized, referencedObjects, cteNames };
}
/** `WITH x AS (…), y AS (…)` — names the statement defines for itself. */
function collectCteNames(stripped: string): string[] {
const names: string[] = [];
const re = new RegExp(`(?:^|[,\\s(])(${IDENT})\\s+as\\s*\\(`, 'gi');
for (const m of stripped.matchAll(re)) names.push(m[1].toLowerCase());
return names;
}
/**
* Every `FROM` / `JOIN` target, resolved.
*
* Rules, all fail-closed:
* * `(` after FROM/JOIN is a subquery — fine, its own FROM is scanned too.
* * `name(` after FROM/JOIN is a TABLE FUNCTION — refused outright, even if
* the name is not on the deny-list, because an unknown table function is
* precisely the case the deny-list cannot cover.
* * a bare `name` must be a CTE this statement defined.
* * a `db.name` must have `db === allowedCatalog` and `name` in the
* allow-list. Three-part `a.b.c` names are refused.
*/
function collectTableReferences(stripped: string, cteNames: string[], opts: StatementGuardOptions): string[] {
const allowed = new Set(opts.allowedObjects.map((o) => o.toLowerCase()));
const ctes = new Set(cteNames);
const found = new Set<string>();
const re = /(^|[^A-Za-z0-9_])(from|join)([^A-Za-z0-9_]|$)/gi;
let m: RegExpExecArray | null;
while ((m = re.exec(stripped)) !== null) {
const after = stripped.slice(m.index + m[0].length - 1);
const target = /^\s*([A-Za-z_][A-Za-z0-9_.]*|\()/.exec(after);
if (!target) deny('could not resolve a FROM/JOIN target');
const token = target[1];
if (token === '(') continue; // subquery; its own FROM is scanned by this loop
const rest = after.slice(target[0].length);
if (/^\s*\(/.test(rest)) {
deny(`table function "${token}" is not permitted in an analyst session`);
}
const parts = token.split('.');
if (parts.length === 1) {
if (!ctes.has(parts[0].toLowerCase())) {
deny(
`unqualified table "${parts[0]}" — an analyst statement must name ` +
`${opts.allowedCatalog}.<object>, or a CTE it defined itself`,
);
}
continue;
}
if (parts.length !== 2 || !QUALIFIED_RE.test(token)) {
deny(`"${token}" is not a valid <catalog>.<object> reference`);
}
const [catalog, object] = parts;
if (catalog.toLowerCase() !== opts.allowedCatalog.toLowerCase()) {
deny(`catalog "${catalog}" is not readable by this session`);
}
if (!allowed.has(object.toLowerCase())) {
deny(`object "${catalog}.${object}" is not readable by this session`);
}
found.add(`${catalog}.${object}`);
}
return [...found];
}
/** Exported for the test corpus and for operator documentation. */
export const GUARD_BANNED_KEYWORDS = BANNED_KEYWORDS;
export const GUARD_BANNED_FUNCTIONS = BANNED_FUNCTIONS;