517 lines
18 KiB
TypeScript
517 lines
18 KiB
TypeScript
/**
|
|
* src/analytics/sql/statement-guard.ts
|
|
* =====================================
|
|
* BI-P017 — ANALYST SQL CONTRACT, layer 2 (statement admission).
|
|
*
|
|
* READ THIS BEFORE TRUSTING IT.
|
|
*
|
|
* This module is a GUARD, not a proof. It is the second of four layers, and
|
|
* it is the WEAKEST of the four, because it reasons about SQL text without
|
|
* being ClickHouse's parser. A guard that reads a dialect it does not
|
|
* implement can always be wrong about some string. The layers below it are
|
|
* the ones that hold when it is:
|
|
*
|
|
* 1. ROLE — only a CL0UD-verified analyst role reaches a session
|
|
* at all (analyst-sql-session.ts).
|
|
* 2. THIS GUARD — statement admission: one read-only statement, no
|
|
* banned construct, every table reference resolved
|
|
* into the tenant's own catalog.
|
|
* 3. GRANTS — the session authenticates as a per-tenant ClickHouse
|
|
* user with `readonly = 1` and SELECT granted ONLY on
|
|
* that tenant's catalog. A statement this guard wrongly
|
|
* admits still cannot read a table the user cannot see.
|
|
* 4. VIEW PREDICATE — the tenant catalog contains VIEWS, never base tables,
|
|
* and each view bakes `WHERE tenant_id = '<tenant>'`
|
|
* into its own definition. A row policy on the base
|
|
* tables repeats it server-side.
|
|
*
|
|
* tests/integration/analyst-sql-clickhouse.integration.test.ts runs the
|
|
* adversarial corpus against a live ClickHouse with layer 2 REMOVED, and
|
|
* requires layers 3 and 4 to refuse it anyway. A defence-in-depth claim that
|
|
* has never been tested with a layer removed is not evidence — this estate
|
|
* has shipped several.
|
|
*
|
|
* DESIGN NOTES that are load-bearing:
|
|
*
|
|
* * Identifiers must resolve to an ALLOW-LIST. The deny-lists below are a
|
|
* second opinion and a better error message, never the mechanism. Every
|
|
* table reference is checked positively against the tenant's catalog.
|
|
* * Outside string literals the statement must be ASCII. Homoglyph and
|
|
* zero-width tricks against keyword matching stop being expressible.
|
|
* * The FROM/JOIN scan is CONSERVATIVE where it is unsure, and that costs
|
|
* something worth naming: ClickHouse's `ARRAY JOIN arr` is a legitimate
|
|
* construct whose target is a COLUMN, not a table, and this guard refuses
|
|
* it because the token after JOIN is neither a CTE nor a qualified object.
|
|
* A false refusal an analyst can see and report is the right side of that
|
|
* trade; a false ADMISSION is not. Supporting it means teaching the scan
|
|
* which JOIN flavours take a non-table target, which is the direction that
|
|
* grows holes, so it stays refused until someone needs it and can add it
|
|
* WITH a corpus entry proving the narrowing did not open anything else.
|
|
* * Comments and string literals are removed BEFORE any keyword scan, using
|
|
* ClickHouse's own lexical rules (`--` and `#` to end of line, `/* *\/`
|
|
* non-nesting, `'` with backslash and doubled-quote escapes, `"` and
|
|
* backtick as identifier quotes).
|
|
*/
|
|
import { SiaxError } from '../../platform/errors';
|
|
|
|
export interface StatementGuardOptions {
|
|
/**
|
|
* The ONLY database name a table reference may carry. This is the tenant's
|
|
* own catalog of pre-filtered views, not the shared analytics database.
|
|
*/
|
|
allowedCatalog: string;
|
|
/** Object names inside that catalog the analyst may read. */
|
|
allowedObjects: readonly string[];
|
|
/** Hard ceiling on statement length; a megabyte of SQL is an attack, not a query. */
|
|
maxLength?: number;
|
|
}
|
|
|
|
export interface GuardedStatement {
|
|
/** The statement as submitted, unchanged. */
|
|
raw: string;
|
|
/** Comments and literals removed — what the keyword scan actually saw. */
|
|
stripped: string;
|
|
/** Literals replaced by `?`, for statement history that is not a PII dump. */
|
|
normalized: string;
|
|
/** Fully-qualified objects the statement reads, resolved against the catalog. */
|
|
referencedObjects: string[];
|
|
/** Common-table-expression names defined by the statement itself. */
|
|
cteNames: string[];
|
|
}
|
|
|
|
const DEFAULT_MAX_LENGTH = 20_000;
|
|
|
|
/**
|
|
* Statement-level constructs that make a statement not-a-read. Matched as
|
|
* whole words against the stripped text.
|
|
*
|
|
* `SETTINGS` is here because a trailing `SETTINGS max_rows_to_read = …` is the
|
|
* obvious way to try to lift the governance ceiling from inside the query.
|
|
* `FORMAT` is here because the session appends its own; a caller-supplied one
|
|
* is either a duplicate or an attempt at `FORMAT ...` side effects.
|
|
*/
|
|
const BANNED_KEYWORDS = [
|
|
'insert',
|
|
'update',
|
|
'delete',
|
|
'drop',
|
|
'alter',
|
|
'create',
|
|
'truncate',
|
|
'rename',
|
|
'attach',
|
|
'detach',
|
|
'replace',
|
|
'optimize',
|
|
'grant',
|
|
'revoke',
|
|
'kill',
|
|
'system',
|
|
'use',
|
|
'set',
|
|
'settings',
|
|
'exchange',
|
|
'move',
|
|
'undrop',
|
|
'backup',
|
|
'restore',
|
|
'outfile',
|
|
'infile',
|
|
'format',
|
|
'watch',
|
|
'check',
|
|
'explain',
|
|
'show',
|
|
'exists',
|
|
'call',
|
|
'execute',
|
|
'declare',
|
|
'begin',
|
|
'commit',
|
|
'rollback',
|
|
] as const;
|
|
|
|
/**
|
|
* Functions that read something other than a table the grants control:
|
|
* the filesystem, the network, another server, a dictionary, an executable.
|
|
* ClickHouse table functions are the whole reason an analyst SQL surface is
|
|
* dangerous — `SELECT * FROM url('http://…')` needs no table privilege at all.
|
|
*/
|
|
const BANNED_FUNCTIONS = [
|
|
'file',
|
|
'url',
|
|
'urlcluster',
|
|
's3',
|
|
's3cluster',
|
|
'gcs',
|
|
'azureblobstorage',
|
|
'remote',
|
|
'remotesecure',
|
|
'cluster',
|
|
'clusterallreplicas',
|
|
'hdfs',
|
|
'hdfscluster',
|
|
'mysql',
|
|
'postgresql',
|
|
'mongodb',
|
|
'jdbc',
|
|
'odbc',
|
|
'sqlite',
|
|
'redis',
|
|
'deltalake',
|
|
'hudi',
|
|
'iceberg',
|
|
'executable',
|
|
'input',
|
|
'generaterandom',
|
|
'numbers_mt',
|
|
'merge',
|
|
'view',
|
|
'viewifpermitted',
|
|
'loop',
|
|
'fuzzjson',
|
|
'fuzzquery',
|
|
'dictget',
|
|
'dictgetordefault',
|
|
'dictgetornull',
|
|
'dictgetstring',
|
|
'dictgetuint64',
|
|
'dictgethierarchy',
|
|
'dictisin',
|
|
'dicthas',
|
|
'dictgetchildren',
|
|
'dictgetdescendants',
|
|
'filesystemavailable',
|
|
'filesystemfree',
|
|
'filesystemcapacity',
|
|
'currentdatabase',
|
|
'currentuser',
|
|
'hostname',
|
|
'fqdn',
|
|
'uptime',
|
|
'version',
|
|
'getsetting',
|
|
'getmacro',
|
|
'shardnum',
|
|
'shardcount',
|
|
'buildid',
|
|
'servertimezone',
|
|
'readwkt',
|
|
'structurefromjson',
|
|
's3queue',
|
|
'timeseries',
|
|
'mergetreeindex',
|
|
] as const;
|
|
|
|
/**
|
|
* Constructs that a whole-word keyword ban gets WRONG, so they get a pattern.
|
|
*
|
|
* `DESC` is the case that proves the point: banning the word breaks every
|
|
* `ORDER BY x DESC` an analyst will ever write, and the test corpus caught it
|
|
* immediately. What is actually dangerous is `DESCRIBE`/`DESC` used as a
|
|
* STATEMENT — including inside a subquery, where ClickHouse accepts
|
|
* `SELECT * FROM (DESCRIBE TABLE x)` and the FROM-scan sees only a `(`.
|
|
*/
|
|
const BANNED_PATTERNS: Array<{ re: RegExp; reason: string }> = [
|
|
{ re: /\bdesc(ribe)?\s+(table|\()/i, reason: 'DESCRIBE is not permitted in a read-only analyst session' },
|
|
{ re: /\bcurrent_user\b/i, reason: 'CURRENT_USER is not permitted in an analyst session' },
|
|
{ re: /\binto\s+outfile\b/i, reason: 'INTO OUTFILE is not permitted in an analyst session' },
|
|
];
|
|
|
|
/** Names that are catalogs an analyst may never reach, whatever the grants say. */
|
|
const BANNED_CATALOGS = [
|
|
'system',
|
|
'information_schema',
|
|
'INFORMATION_SCHEMA',
|
|
'default',
|
|
'opt_analytics',
|
|
'opt_scraped',
|
|
];
|
|
|
|
const IDENT = '[A-Za-z_][A-Za-z0-9_]*';
|
|
const QUALIFIED_RE = new RegExp(`^${IDENT}\\.${IDENT}$`);
|
|
|
|
function deny(detail: string): never {
|
|
throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics.sql', detail);
|
|
}
|
|
|
|
function invalid(detail: string): never {
|
|
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.sql', detail);
|
|
}
|
|
|
|
interface StripResult {
|
|
stripped: string;
|
|
normalized: string;
|
|
/** Identifier tokens that were quoted, kept so they can still be resolved. */
|
|
hadQuotedIdentifier: boolean;
|
|
}
|
|
|
|
/**
|
|
* Remove comments and string literals using ClickHouse's lexical rules.
|
|
*
|
|
* Two outputs, deliberately: `stripped` keeps the literal DELIMITERS so the
|
|
* token stream still parses (`'x'` becomes `''`), and `normalized` replaces
|
|
* each literal with `?` so a statement history is searchable without being a
|
|
* copy of the tenant's data.
|
|
*
|
|
* An unterminated literal or comment is a REJECTION. Silently treating the
|
|
* rest of the statement as a string is exactly how a guard and an engine come
|
|
* to disagree about where the query ends.
|
|
*/
|
|
export function stripSqlText(sql: string): StripResult {
|
|
let stripped = '';
|
|
let normalized = '';
|
|
let hadQuotedIdentifier = false;
|
|
let i = 0;
|
|
const n = sql.length;
|
|
|
|
while (i < n) {
|
|
const c = sql[i];
|
|
const next = sql[i + 1];
|
|
|
|
// -- line comment
|
|
if (c === '-' && next === '-') {
|
|
while (i < n && sql[i] !== '\n') i += 1;
|
|
stripped += ' ';
|
|
normalized += ' ';
|
|
continue;
|
|
}
|
|
// # line comment (ClickHouse accepts `#` and `#!`)
|
|
if (c === '#') {
|
|
while (i < n && sql[i] !== '\n') i += 1;
|
|
stripped += ' ';
|
|
normalized += ' ';
|
|
continue;
|
|
}
|
|
// /* block comment */ — ClickHouse does NOT nest these
|
|
if (c === '/' && next === '*') {
|
|
const end = sql.indexOf('*/', i + 2);
|
|
if (end === -1) invalid('unterminated block comment');
|
|
i = end + 2;
|
|
stripped += ' ';
|
|
normalized += ' ';
|
|
continue;
|
|
}
|
|
// 'string literal'
|
|
if (c === "'") {
|
|
i += 1;
|
|
while (i < n) {
|
|
if (sql[i] === '\\') {
|
|
i += 2;
|
|
continue;
|
|
}
|
|
if (sql[i] === "'") {
|
|
if (sql[i + 1] === "'") {
|
|
i += 2;
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
i += 1;
|
|
}
|
|
if (i >= n) invalid('unterminated string literal');
|
|
i += 1; // closing quote
|
|
stripped += "''";
|
|
normalized += '?';
|
|
continue;
|
|
}
|
|
// "quoted identifier" / `quoted identifier`
|
|
if (c === '"' || c === '`') {
|
|
const quote = c;
|
|
let ident = '';
|
|
i += 1;
|
|
while (i < n) {
|
|
if (sql[i] === '\\') {
|
|
ident += sql[i + 1] ?? '';
|
|
i += 2;
|
|
continue;
|
|
}
|
|
if (sql[i] === quote) {
|
|
if (sql[i + 1] === quote) {
|
|
ident += quote;
|
|
i += 2;
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
ident += sql[i];
|
|
i += 1;
|
|
}
|
|
if (i >= n) invalid('unterminated quoted identifier');
|
|
i += 1;
|
|
hadQuotedIdentifier = true;
|
|
// The identifier is put back UNQUOTED so the table-reference scan sees
|
|
// it exactly as it would see a bare one. A quoted identifier that does
|
|
// not survive that scan is refused like any other.
|
|
if (!new RegExp(`^${IDENT}$`).test(ident)) {
|
|
deny(`quoted identifier "${ident.slice(0, 32)}" is not a plain name`);
|
|
}
|
|
stripped += ident;
|
|
normalized += ident;
|
|
continue;
|
|
}
|
|
stripped += c;
|
|
normalized += c;
|
|
i += 1;
|
|
}
|
|
return { stripped, normalized, hadQuotedIdentifier };
|
|
}
|
|
|
|
/**
|
|
* Admit a statement, or refuse it.
|
|
*
|
|
* Returns the parsed view of what was admitted; the caller (the session) is
|
|
* responsible for actually running it under the restricted user.
|
|
*/
|
|
export function guardAnalystStatement(sql: string, opts: StatementGuardOptions): GuardedStatement {
|
|
const maxLength = opts.maxLength ?? DEFAULT_MAX_LENGTH;
|
|
if (typeof sql !== 'string') invalid('statement must be a string');
|
|
const raw = sql.trim();
|
|
if (!raw) invalid('statement is empty');
|
|
if (raw.length > maxLength) invalid(`statement exceeds ${maxLength} characters`);
|
|
if (!opts.allowedCatalog || !new RegExp(`^${IDENT}$`).test(opts.allowedCatalog)) {
|
|
// A misconfigured session must not degrade to "any catalog".
|
|
deny('session has no valid tenant catalog');
|
|
}
|
|
if (BANNED_CATALOGS.includes(opts.allowedCatalog)) {
|
|
deny(`session catalog "${opts.allowedCatalog}" is a reserved catalog`);
|
|
}
|
|
if (opts.allowedObjects.length === 0) {
|
|
deny('session exposes no readable objects');
|
|
}
|
|
|
|
const { stripped, normalized } = stripSqlText(raw);
|
|
|
|
// ── ASCII-only outside literals ────────────────────────────────────────
|
|
// Homoglyphs, zero-width joiners and RTL overrides stop being expressible
|
|
// against the keyword scan. Values are unaffected: they live in literals,
|
|
// which have already been removed at this point.
|
|
// Written as a code-point scan rather than a character class: a regex of
|
|
// control characters is unreadable, trips `no-control-regex`, and is the
|
|
// kind of expression a reviewer nods at without checking. Permitted: tab,
|
|
// LF, CR and printable ASCII. Everything else, NUL included, is refused.
|
|
for (let i = 0; i < stripped.length; i += 1) {
|
|
const code = stripped.charCodeAt(i);
|
|
const printable = code >= 0x20 && code <= 0x7e;
|
|
const allowedWhitespace = code === 0x09 || code === 0x0a || code === 0x0d;
|
|
if (!printable && !allowedWhitespace) {
|
|
deny(
|
|
`statement contains a disallowed character (U+${code.toString(16).toUpperCase().padStart(4, '0')}) ` +
|
|
`outside a string literal`,
|
|
);
|
|
}
|
|
}
|
|
|
|
// ── exactly one statement ──────────────────────────────────────────────
|
|
const withoutTrailingSemi = stripped.replace(/;\s*$/, '');
|
|
if (withoutTrailingSemi.includes(';')) {
|
|
deny('multiple statements are not allowed in an analyst SQL session');
|
|
}
|
|
|
|
const lower = ` ${withoutTrailingSemi.toLowerCase().replace(/\s+/g, ' ').trim()} `;
|
|
|
|
// ── must be a read ─────────────────────────────────────────────────────
|
|
if (!/^\s*(select|with)\b/i.test(withoutTrailingSemi.trim())) {
|
|
deny('analyst SQL sessions accept SELECT and WITH statements only');
|
|
}
|
|
|
|
for (const kw of BANNED_KEYWORDS) {
|
|
if (new RegExp(`(^|[^A-Za-z0-9_])${kw}([^A-Za-z0-9_]|$)`).test(lower)) {
|
|
deny(`"${kw.toUpperCase()}" is not permitted in a read-only analyst session`);
|
|
}
|
|
}
|
|
|
|
for (const { re, reason } of BANNED_PATTERNS) {
|
|
if (re.test(lower)) deny(reason);
|
|
}
|
|
|
|
// ── banned functions, matched as `name(` ───────────────────────────────
|
|
const calls = [...lower.matchAll(new RegExp(`([a-z_][a-z0-9_]*)\\s*\\(`, 'g'))].map((m) => m[1]);
|
|
for (const call of calls) {
|
|
if ((BANNED_FUNCTIONS as readonly string[]).includes(call)) {
|
|
deny(`function "${call}" is not permitted in an analyst session`);
|
|
}
|
|
}
|
|
|
|
// ── table references: positively resolved against the catalog ──────────
|
|
const cteNames = collectCteNames(withoutTrailingSemi);
|
|
const referencedObjects = collectTableReferences(withoutTrailingSemi, cteNames, opts);
|
|
|
|
if (referencedObjects.length === 0) {
|
|
// A SELECT with no FROM (`SELECT 1`) is harmless, but so is refusing it,
|
|
// and "no table reference" is also what a successful parser evasion looks
|
|
// like. Refuse: an analyst session exists to read tables.
|
|
deny('statement references no readable object in the tenant catalog');
|
|
}
|
|
|
|
return { raw, stripped, normalized, referencedObjects, cteNames };
|
|
}
|
|
|
|
/** `WITH x AS (…), y AS (…)` — names the statement defines for itself. */
|
|
function collectCteNames(stripped: string): string[] {
|
|
const names: string[] = [];
|
|
const re = new RegExp(`(?:^|[,\\s(])(${IDENT})\\s+as\\s*\\(`, 'gi');
|
|
for (const m of stripped.matchAll(re)) names.push(m[1].toLowerCase());
|
|
return names;
|
|
}
|
|
|
|
/**
|
|
* Every `FROM` / `JOIN` target, resolved.
|
|
*
|
|
* Rules, all fail-closed:
|
|
* * `(` after FROM/JOIN is a subquery — fine, its own FROM is scanned too.
|
|
* * `name(` after FROM/JOIN is a TABLE FUNCTION — refused outright, even if
|
|
* the name is not on the deny-list, because an unknown table function is
|
|
* precisely the case the deny-list cannot cover.
|
|
* * a bare `name` must be a CTE this statement defined.
|
|
* * a `db.name` must have `db === allowedCatalog` and `name` in the
|
|
* allow-list. Three-part `a.b.c` names are refused.
|
|
*/
|
|
function collectTableReferences(stripped: string, cteNames: string[], opts: StatementGuardOptions): string[] {
|
|
const allowed = new Set(opts.allowedObjects.map((o) => o.toLowerCase()));
|
|
const ctes = new Set(cteNames);
|
|
const found = new Set<string>();
|
|
|
|
const re = /(^|[^A-Za-z0-9_])(from|join)([^A-Za-z0-9_]|$)/gi;
|
|
let m: RegExpExecArray | null;
|
|
while ((m = re.exec(stripped)) !== null) {
|
|
const after = stripped.slice(m.index + m[0].length - 1);
|
|
const target = /^\s*([A-Za-z_][A-Za-z0-9_.]*|\()/.exec(after);
|
|
if (!target) deny('could not resolve a FROM/JOIN target');
|
|
const token = target[1];
|
|
if (token === '(') continue; // subquery; its own FROM is scanned by this loop
|
|
|
|
const rest = after.slice(target[0].length);
|
|
if (/^\s*\(/.test(rest)) {
|
|
deny(`table function "${token}" is not permitted in an analyst session`);
|
|
}
|
|
|
|
const parts = token.split('.');
|
|
if (parts.length === 1) {
|
|
if (!ctes.has(parts[0].toLowerCase())) {
|
|
deny(
|
|
`unqualified table "${parts[0]}" — an analyst statement must name ` +
|
|
`${opts.allowedCatalog}.<object>, or a CTE it defined itself`,
|
|
);
|
|
}
|
|
continue;
|
|
}
|
|
if (parts.length !== 2 || !QUALIFIED_RE.test(token)) {
|
|
deny(`"${token}" is not a valid <catalog>.<object> reference`);
|
|
}
|
|
const [catalog, object] = parts;
|
|
if (catalog.toLowerCase() !== opts.allowedCatalog.toLowerCase()) {
|
|
deny(`catalog "${catalog}" is not readable by this session`);
|
|
}
|
|
if (!allowed.has(object.toLowerCase())) {
|
|
deny(`object "${catalog}.${object}" is not readable by this session`);
|
|
}
|
|
found.add(`${catalog}.${object}`);
|
|
}
|
|
return [...found];
|
|
}
|
|
|
|
/** Exported for the test corpus and for operator documentation. */
|
|
export const GUARD_BANNED_KEYWORDS = BANNED_KEYWORDS;
|
|
export const GUARD_BANNED_FUNCTIONS = BANNED_FUNCTIONS;
|