0pt/tests/integration/analytics-tenant-isolation.integration.test.ts

/**
 * BI-P011 — CLICKHOUSE TENANT ISOLATION, proven against a LIVE server.
 *
 * The shape of this proof is the estate's own convention for isolation work:
 * DEMONSTRATE THE HOLE WITHOUT THE FIX, then demonstrate it CLOSED WITH the
 * fix. A suite that only exercises the fixed path proves that the fixed path
 * works, which is not the same claim.
 *
 * So each layer is tested twice:
 *
 *   layer 2 (0pt query compiler forces tenant)
 *     hole:   a hand-written statement WITHOUT the predicate — the same query
 *             minus the forcing — returns another tenant's rows. This is the
 *             literal regression a future refactor would cause.
 *     closed: the compiled statement returns only this tenant's rows.
 *
 *   layer 3 (restricted database users + row policies)
 *     hole:   the UNRESTRICTED admin user, issuing a statement with NO tenant
 *             predicate at all, sees every tenant.
 *     closed: the RESTRICTED per-tenant user, issuing the SAME statement,
 *             sees only its own tenant — the server filters, so the result
 *             holds even if layer 2 were removed entirely.
 *
 * Gated on CLICKHOUSE_URL and SKIPS HONESTLY when unset, matching
 * tests/integration/analytics-clickhouse.integration.test.ts. CI's
 * `integration` job sets it from a service container and hard-fails if the
 * suites skip, so this is not a self-skipping green in practice.
 */
import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { ClickHouseAnalyticsEngine } from "@/src/analytics/engine/clickhouse-engine";
import {
  compileMetricQuery,
  dimensionCatalogFrom,
  restrictedTenantUserDdl,
} from "@/src/analytics/engine/query-compiler";
import { ANALYTICS_TABLES, sortingKeySql } from "@/src/analytics/engine/schema";
import { DEFAULT_BUDGETS } from "@/src/analytics/engine/resource-governance";
import type { AnalyticsIdentity } from "@/src/analytics/engine/siax-analytics-engine";


/**
 * Raw events carry a 30-day TTL (src/analytics/engine/schema.ts). Fixed
 * historical timestamps silently expire and make live-ClickHouse suites read
 * 0 rows, so seed rows are stamped relative to now (UTC, ClickHouse
 * DateTime64(3) text form).
 */
const SEED_EVENT_TIME = new Date(Date.now() - 2 * 3600_000).toISOString().replace("T", " ").replace("Z", "");

const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL;
const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH;
const run = CLICKHOUSE_URL ? describe : describe.skip;

const DB = `opt_isolation_${Date.now().toString(36)}`;
const RESTRICTED_USER = `opt_iso_${Date.now().toString(36)}`;
const RESTRICTED_PW = `iso_pw_${Math.random().toString(36).slice(2, 12)}`;

const ALPHA: AnalyticsIdentity = { tenant_id: "t_alpha", project_id: "p_main", principal_id: "usr_alpha" };
const BETA: AnalyticsIdentity = { tenant_id: "t_beta", project_id: "p_main", principal_id: "usr_beta" };

const catalog = dimensionCatalogFrom({ severity: "severity" });
const ALL_TABLES: readonly string[] = ANALYTICS_TABLES;

/** Raw HTTP against the live server, as a named user. */
async function chQuery(
  sql: string,
  auth: string | undefined,
  params: Record<string, string> = {},
  settings: Record<string, string> = {},
  body?: string,
): Promise<{ ok: boolean; status: number; text: string }> {
  const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid");
  url.searchParams.set("query", sql);
  for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v);
  for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
  const headers: Record<string, string> = {};
  if (auth) headers["Authorization"] = `Basic ${Buffer.from(auth).toString("base64")}`;
  const res = await fetch(url.toString(), { method: "POST", headers, body: body ?? "" });
  return { ok: res.ok, status: res.status, text: await res.text() };
}

function rowsOf(text: string): { grp?: string; value: number }[] {
  if (!text.trim()) return [];
  const parsed = JSON.parse(text) as { data?: { grp?: string; value: number | string }[] };
  return (parsed.data ?? []).map((r) => ({ ...(r.grp !== undefined ? { grp: r.grp } : {}), value: Number(r.value) }));
}

let engine: ClickHouseAnalyticsEngine;

run("BI-P011 tenant isolation against a LIVE ClickHouse", () => {
  beforeAll(async () => {
    engine = new ClickHouseAnalyticsEngine({
      baseUrl: CLICKHOUSE_URL!,
      basicAuth: CLICKHOUSE_AUTH,
      database: DB,
      dimensionCatalogs: { t_alpha: { severity: "severity" }, t_beta: { severity: "severity" } },
    });
    const health = await engine.health();
    expect(health.healthy, `ClickHouse not healthy: ${health.detail ?? ""}`).toBe(true);
    await engine.ensureSchema();

    // Two tenants, deliberately different magnitudes so a leak is unmistakable
    // in the NUMBER, not just in the row count.
    const rows: string[] = [];
    for (let i = 0; i < 50; i++) {
      rows.push(
        JSON.stringify({
          tenant_id: "t_alpha",
          project_id: "p_main",
          dataset_id: "ds_shared",
          metric: "rows_ingested",
          event_id: `a${i}`,
          event_time: SEED_EVENT_TIME,
          value: 1,
          dim_names: ["severity"],
          dim_values: [i % 2 === 0 ? "warning" : "critical"],
        }),
      );
      rows.push(
        JSON.stringify({
          tenant_id: "t_beta",
          project_id: "p_main",
          dataset_id: "ds_shared",
          metric: "rows_ingested",
          event_id: `b${i}`,
          event_time: SEED_EVENT_TIME,
          value: 1000,
          dim_names: ["severity"],
          dim_values: ["critical"],
        }),
      );
    }
    const ins = await chQuery(
      `INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`,
      CLICKHOUSE_AUTH,
      {},
      {},
      rows.join("\n"),
    );
    expect(ins.ok, `seed insert failed: ${ins.text.slice(0, 200)}`).toBe(true);
  }, 120_000);

  afterAll(async () => {
    if (!CLICKHOUSE_URL) return;
    // Disposable by construction: drop the user, its policies, and the DB.
    for (const table of ANALYTICS_TABLES) {
      await chQuery(`DROP ROW POLICY IF EXISTS ${RESTRICTED_USER}_${table}_rp ON ${DB}.${table}`, CLICKHOUSE_AUTH);
    }
    await chQuery(`DROP USER IF EXISTS ${RESTRICTED_USER}`, CLICKHOUSE_AUTH);
    await chQuery(`DROP DATABASE IF EXISTS ${DB}`, CLICKHOUSE_AUTH);
  }, 60_000);

  // ── layer 2: the compiler ────────────────────────────────────────────────

  it("HOLE, layer 2 REMOVED: the same query WITHOUT the forced predicate returns t_beta's rows to t_alpha", async () => {
    // This is the exact statement the compiler would produce if someone
    // deleted the forcing — proof that the predicate is doing real work
    // against real data, not decorating a query that was safe anyway.
    const unsafeSql =
      `SELECT sum(value) AS value FROM ${DB}.opt_analytics_event ` +
      `WHERE metric = {metric:String} LIMIT 1 FORMAT JSON`;
    const res = await chQuery(unsafeSql, CLICKHOUSE_AUTH, { param_metric: "rows_ingested" });
    expect(res.ok).toBe(true);
    // 50*1 (alpha) + 50*1000 (beta) = 50050 — both tenants, one result.
    expect(rowsOf(res.text)[0].value).toBe(50050);
  });

  it("CLOSED, layer 2 PRESENT: the compiled query returns t_alpha's 50 and never t_beta's 50000", async () => {
    const result = await engine.executeMetric(ALPHA, { metric: "rows_ingested", aggregate: "sum" });
    expect(result.rows[0].value).toBe(50);
    const beta = await engine.executeMetric(BETA, { metric: "rows_ingested", aggregate: "sum" });
    expect(beta.rows[0].value).toBe(50_000);
  });

  it("tenant A querying tenant B's dataset id gets ZERO rows, not B's rows", async () => {
    const result = await engine.executeMetric(ALPHA, {
      metric: "rows_ingested",
      aggregate: "count",
      dataset_id: "ds_shared",
    });
    expect(result.rows[0].value).toBe(50); // only alpha's 50, though beta shares the dataset id
  });

  it("a forged dataset id yields an empty result, never an error that leaks existence", async () => {
    const result = await engine.executeMetric(ALPHA, {
      metric: "rows_ingested",
      aggregate: "count",
      dataset_id: "ds_that_does_not_exist_or_belongs_to_someone_else",
    });
    expect(result.rows[0].value).toBe(0);
  });

  it("a tenant with no data at all sees zero, not everyone's aggregate", async () => {
    const ghost: AnalyticsIdentity = { tenant_id: "t_gamma", project_id: "p_main", principal_id: "usr_g" };
    const result = await engine.executeMetric(ghost, { metric: "rows_ingested", aggregate: "count" });
    expect(result.rows[0].value).toBe(0);
  });

  it("group-by results carry only this tenant's dimension values", async () => {
    const result = await engine.executeMetric(ALPHA, {
      metric: "rows_ingested",
      aggregate: "count",
      group_by: ["severity"],
    });
    const total = result.rows.reduce((a, r) => a + r.value, 0);
    expect(total).toBe(50);
    // t_beta's rows are all "critical" with value 1000; if any leaked, the
    // critical bucket would exceed alpha's 25.
    const critical = result.rows.find((r) => r.groups?.severity === "critical");
    expect(critical?.value).toBe(25);
  });

  it("SQL/dimension/filter injection through the live path changes nothing", async () => {
    await expect(
      engine.executeMetric(ALPHA, {
        metric: "rows_ingested",
        aggregate: "sum",
        group_by: ["severity'; SELECT tenant_id FROM system.tables; --"],
      }),
    ).rejects.toMatchObject({ code: "INVALID_REQUEST" });

    // A hostile VALUE reaches the server as a bound parameter and simply
    // matches nothing.
    const injected = await engine.executeMetric(ALPHA, {
      metric: "rows_ingested",
      aggregate: "count",
      filters: { severity: "warning' OR 1=1 --" },
    });
    expect(injected.rows[0].value).toBe(0);

    // And the table is still there afterwards.
    const still = await engine.executeMetric(ALPHA, { metric: "rows_ingested", aggregate: "count" });
    expect(still.rows[0].value).toBe(50);
  });

  it("raw SQL attempt: the analyst-SQL surface is refused, not silently executed", async () => {
    await expect(
      engine.executeQuery(ALPHA, { sql: `SELECT * FROM ${DB}.opt_analytics_event`, dataset_ids: [] }),
    ).rejects.toMatchObject({ code: "CAPABILITY_UNAVAILABLE" });
  });

  // ── layer 3: restricted users + row policies ─────────────────────────────

  it("HOLE, layer 3 ABSENT: an unrestricted user with no predicate sees every tenant", async () => {
    const res = await chQuery(
      `SELECT uniqExact(tenant_id) AS value FROM ${DB}.opt_analytics_event FORMAT JSON`,
      CLICKHOUSE_AUTH,
    );
    expect(res.ok).toBe(true);
    expect(rowsOf(res.text)[0].value).toBe(2); // t_alpha AND t_beta
  });

  it("CLOSED, layer 3 PRESENT: the restricted user runs the SAME predicate-free statement and sees ONE tenant", async () => {
    for (const stmt of restrictedTenantUserDdl({
      db: DB,
      tenantId: "t_alpha",
      userName: RESTRICTED_USER,
      password: RESTRICTED_PW,
      tables: ALL_TABLES,
    })) {
      const r = await chQuery(stmt, CLICKHOUSE_AUTH);
      expect(r.ok, `DDL failed: ${stmt.slice(0, 80)} -> ${r.text.slice(0, 200)}`).toBe(true);
    }

    const restrictedAuth = `${RESTRICTED_USER}:${RESTRICTED_PW}`;
    // The IDENTICAL statement that leaked above, now as the restricted user.
    const res = await chQuery(
      `SELECT uniqExact(tenant_id) AS value FROM ${DB}.opt_analytics_event FORMAT JSON`,
      restrictedAuth,
    );
    expect(res.ok, res.text.slice(0, 300)).toBe(true);
    expect(rowsOf(res.text)[0].value).toBe(1);

    const sum = await chQuery(
      `SELECT sum(value) AS value FROM ${DB}.opt_analytics_event FORMAT JSON`,
      restrictedAuth,
    );
    expect(rowsOf(sum.text)[0].value).toBe(50); // alpha's total, not 50050
  });

  it("the restricted user cannot write, and cannot reach a table it was not granted", async () => {
    const restrictedAuth = `${RESTRICTED_USER}:${RESTRICTED_PW}`;
    const write = await chQuery(
      `INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`,
      restrictedAuth,
      {},
      {},
      JSON.stringify({ tenant_id: "t_beta", project_id: "p", dataset_id: "d", metric: "m", event_id: "x", event_time: SEED_EVENT_TIME, value: 1, dim_names: [], dim_values: [] }),
    );
    expect(write.ok).toBe(false);

    const foreign = await chQuery(`SELECT count() FROM system.users FORMAT JSON`, restrictedAuth);
    expect(foreign.ok).toBe(false);
  });

  // ── the structural precondition both layers rely on ──────────────────────

  it("every analytics table's sorting key LEADS with tenant_id (what makes the predicate cheap)", async () => {
    for (const table of ALL_TABLES) {
      const res = await chQuery(sortingKeySql(DB, table), CLICKHOUSE_AUTH);
      expect(res.ok).toBe(true);
      expect(res.text.trim().startsWith("tenant_id"), `${table} sorting key: ${res.text.trim()}`).toBe(true);
    }
  });

  it("the tenant value never appears in the statement text the server receives", async () => {
    const compiled = compileMetricQuery(
      { tenant_id: "t_alpha", metric: "rows_ingested", aggregate: "sum" },
      DB,
      catalog,
      DEFAULT_BUDGETS.INTERACTIVE.maxResultRows,
    );
    expect(compiled.sql).not.toContain("t_alpha");
    const res = await chQuery(compiled.sql, CLICKHOUSE_AUTH, compiled.params);
    expect(res.ok).toBe(true);
    expect(rowsOf(res.text)[0].value).toBe(50);
  });
});