0pt/tests/integration/analytics-resource-governance.integration.test.ts

/**
 * BI-P012 DoD, against a LIVE ClickHouse:
 *
 *   "Intentional query bomb is terminated without degrading unrelated tenant
 *    beyond declared SLO."
 *
 * Two halves, and the second is the one that is usually skipped:
 *
 *   1. TERMINATED — a deliberately enormous query is refused by the server,
 *      with the ClickHouse limit that stopped it identified by code. Each
 *      limit BI-P012 names is fired individually, so "some limit stopped it"
 *      is never mistaken for "the limit I claimed stopped it".
 *   2. UNRELATED TENANT NOT DEGRADED — a second tenant's interactive query is
 *      timed WHILE the bomb is in flight and compared against a declared SLO,
 *      not merely observed to "still work".
 */
import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { ClickHouseAnalyticsEngine } from "@/src/analytics/engine/clickhouse-engine";
import { DEFAULT_BUDGETS } from "@/src/analytics/engine/resource-governance";
import type { AnalyticsIdentity } from "@/src/analytics/engine/siax-analytics-engine";


/**
 * Raw events carry a 30-day TTL (src/analytics/engine/schema.ts). Fixed
 * historical timestamps silently expire and make live-ClickHouse suites read
 * 0 rows, so seed rows are stamped relative to now (UTC, ClickHouse
 * DateTime64(3) text form).
 */
const SEED_EVENT_TIME = new Date(Date.now() - 2 * 3600_000).toISOString().replace("T", " ").replace("Z", "");

const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL;
const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH;
const run = CLICKHOUSE_URL ? describe : describe.skip;

const DB = `opt_gov_${Date.now().toString(36)}`;

/**
 * The SLO this test asserts against. Declared as a constant so the claim is
 * falsifiable rather than "it still seemed to work".
 *
 * 5000 ms is deliberately generous for a 500-row lookup (it measures ~30 ms
 * on an idle server). It is set for the SHARED CI runner, not for a quiet
 * laptop: a tenant that had actually been starved by the bomb would take
 * seconds or fail outright, so the assertion still discriminates, while a
 * merely busy runner does not turn this gate into a flake.
 */
const UNRELATED_TENANT_SLO_MS = 5000;

const VICTIM: AnalyticsIdentity = { tenant_id: "t_victim", project_id: "p_main", principal_id: "usr_v" };

let engine: ClickHouseAnalyticsEngine;

/** Run a statement with explicit settings and report the ClickHouse code. */
async function chRun(
  sql: string,
  settings: Record<string, string> = {},
  body?: string,
): Promise<{ ok: boolean; code?: string; text: string; ms: number }> {
  const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid");
  url.searchParams.set("query", sql);
  for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v);
  const headers: Record<string, string> = {};
  if (CLICKHOUSE_AUTH) headers["Authorization"] = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString("base64")}`;
  const t0 = Date.now();
  const res = await fetch(url.toString(), { method: "POST", headers, body: body ?? "" });
  const text = await res.text();
  return { ok: res.ok, code: /Code: (\d+)/.exec(text)?.[1], text, ms: Date.now() - t0 };
}

run("BI-P012 resource governance against a LIVE ClickHouse", () => {
  beforeAll(async () => {
    engine = new ClickHouseAnalyticsEngine({
      baseUrl: CLICKHOUSE_URL!,
      basicAuth: CLICKHOUSE_AUTH,
      database: DB,
      dimensionCatalogs: { t_victim: { severity: "severity" } },
    });
    await engine.ensureSchema();
    // A small, real dataset for the unrelated tenant.
    const rows = Array.from({ length: 500 }, (_, i) =>
      JSON.stringify({
        tenant_id: "t_victim",
        project_id: "p_main",
        dataset_id: "ds_v",
        metric: "victim_metric",
        event_id: `v${i}`,
        event_time: SEED_EVENT_TIME,
        value: 1,
        dim_names: ["severity"],
        dim_values: ["warning"],
      }),
    ).join("\n");
    const ins = await chRun(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, {}, rows);
    expect(ins.ok, ins.text.slice(0, 200)).toBe(true);
  }, 120_000);

  afterAll(async () => {
    if (CLICKHOUSE_URL) await chRun(`DROP DATABASE IF EXISTS ${DB}`);
  }, 60_000);

  // ── each named limit, fired individually ────────────────────────────────

  it("max_rows_to_read terminates a read bomb (CH code 158 TOO_MANY_ROWS)", async () => {
    const r = await chRun("SELECT count() FROM numbers(1000000000) FORMAT JSON", {
      readonly: "1",
      max_rows_to_read: "1000",
      read_overflow_mode: "throw",
    });
    expect(r.ok).toBe(false);
    expect(r.code).toBe("158");
  });

  it("max_bytes_to_read terminates a scan bomb (CH code 307 TOO_MANY_BYTES)", async () => {
    const r = await chRun("SELECT sum(number) FROM numbers(1000000000) FORMAT JSON", {
      readonly: "1",
      max_bytes_to_read: "1024",
      read_overflow_mode: "throw",
    });
    expect(r.ok).toBe(false);
    expect(r.code).toBe("307");
  });

  it("max_execution_time terminates a slow bomb (CH code 159 TIMEOUT_EXCEEDED)", async () => {
    const r = await chRun("SELECT count() FROM numbers(50000000000) FORMAT JSON", {
      readonly: "1",
      max_execution_time: "1",
      timeout_overflow_mode: "throw",
    });
    expect(r.ok).toBe(false);
    expect(r.code).toBe("159");
    expect(r.ms).toBeLessThan(20_000);
  });

  it("max_result_rows terminates a result bomb and does NOT return a partial result", async () => {
    const r = await chRun("SELECT number FROM numbers(100000) FORMAT JSON", {
      readonly: "1",
      max_result_rows: "10",
      result_overflow_mode: "throw",
    });
    expect(r.ok).toBe(false);
    expect(r.code).toBe("396"); // TOO_MANY_ROWS_OR_BYTES
    // The dangerous alternative: result_overflow_mode=break returns 10 rows
    // with HTTP 200 and no indication of truncation.
    const broken = await chRun("SELECT number FROM numbers(100000) FORMAT JSON", {
      readonly: "1",
      max_result_rows: "10",
      result_overflow_mode: "break",
    });
    // MEASURED on ClickHouse 24.8: break mode answers HTTP 200 with 65409 of
    // 100000 rows (it stops at a block boundary, not at the stated 10) and
    // nothing in the payload says it is short. That is why every SELECT in
    // this engine ships result_overflow_mode=throw.
    expect(broken.ok).toBe(true);
    const brokenRows = (JSON.parse(broken.text) as { data: unknown[] }).data.length;
    expect(brokenRows).toBeGreaterThan(10);
    expect(brokenRows).toBeLessThan(100_000);
  });

  it("max_memory_usage terminates a memory bomb (CH code 241 MEMORY_LIMIT_EXCEEDED)", async () => {
    const r = await chRun(
      "SELECT uniqExact(toString(number)) FROM numbers(50000000) FORMAT JSON",
      { readonly: "1", max_memory_usage: "1000000", max_execution_time: "30" },
    );
    expect(r.ok).toBe(false);
    expect(r.code).toBe("241");
  });

  it("readonly=1 means the query text cannot re-raise its own limits", async () => {
    const r = await chRun(
      "SELECT count() FROM numbers(1000000000) SETTINGS max_rows_to_read = 10000000000 FORMAT JSON",
      { readonly: "1", max_rows_to_read: "1000", read_overflow_mode: "throw" },
    );
    expect(r.ok).toBe(false);
    // 164 = READONLY (changing settings is forbidden); 158 = the limit still
    // applied. Either proves the caller did not win — but NOT a 200.
    expect(["164", "158"]).toContain(r.code);
  });

  it("readonly=1 blocks writes through the query surface", async () => {
    const r = await chRun(`DROP TABLE ${DB}.opt_analytics_event FORMAT JSON`, { readonly: "1" });
    expect(r.ok).toBe(false);
  });

  // ── the half that is usually skipped ────────────────────────────────────

  it("an unrelated tenant stays within its declared SLO WHILE a query bomb is in flight", async () => {
    // Baseline first, so "fast enough" is measured against this server, not
    // against a number someone hoped for.
    const baseline = await engine.executeMetric(VICTIM, { metric: "victim_metric", aggregate: "count" });
    expect(baseline.rows[0].value).toBe(500);
    const baselineMs = baseline.stats.duration_ms;

    // Fire a genuine bomb the server must spend real CPU on. MEASURED: a bare
    // `count()` over numbers() is NOT a bomb — ClickHouse answers
    // numbers(20e9) in ~5s because it can skip the values entirely. Hashing
    // every row cannot be short-circuited, so this one really does burn CPU
    // until the limit kills it.
    const bomb = chRun("SELECT sum(cityHash64(number)) FROM numbers(20000000000) FORMAT JSON", {
      readonly: "1",
      max_execution_time: "8",
      timeout_overflow_mode: "throw",
    });

    // While it burns, run the unrelated tenant's interactive query repeatedly.
    const samples: number[] = [];
    for (let i = 0; i < 5; i++) {
      const r = await engine.executeMetric(VICTIM, { metric: "victim_metric", aggregate: "count" });
      expect(r.rows[0].value).toBe(500); // still CORRECT, not just fast
      samples.push(r.stats.duration_ms);
    }

    const bombResult = await bomb;
    expect(bombResult.ok, "the bomb was NOT terminated").toBe(false);
    expect(bombResult.code).toBe("159");

    const worst = Math.max(...samples);
    expect(
      worst,
      `unrelated tenant degraded: baseline ${baselineMs}ms, worst under load ${worst}ms, SLO ${UNRELATED_TENANT_SLO_MS}ms`,
    ).toBeLessThan(UNRELATED_TENANT_SLO_MS);
  }, 60_000);

  // ── the local admission gate, on the real engine ────────────────────────

  it("the per-tenant concurrency gate refuses the N+1st in-flight query for THAT tenant only", async () => {
    const limit = DEFAULT_BUDGETS.INTERACTIVE.maxConcurrentQueries;
    // Fire limit+4 concurrently; some must be refused with RATE_LIMITED
    // rather than all of them reaching the server.
    const attempts = Array.from({ length: limit + 4 }, () =>
      engine
        .executeMetric(VICTIM, { metric: "victim_metric", aggregate: "count" })
        .then(() => "ok" as const)
        .catch((e: { code?: string }) => e.code ?? "error"),
    );
    const outcomes = await Promise.all(attempts);
    expect(outcomes.filter((o) => o === "RATE_LIMITED").length).toBeGreaterThan(0);
    expect(outcomes.filter((o) => o === "ok").length).toBeGreaterThan(0);

    // A different tenant is unaffected by the noisy one's exhausted slots.
    const other = await engine.executeMetric(
      { tenant_id: "t_quiet", project_id: "p_main", principal_id: "usr_q" },
      { metric: "victim_metric", aggregate: "count" },
    );
    expect(other.rows[0].value).toBe(0); // its own (empty) data, and no error
  }, 60_000);
});