0pt/tests/analytics/analyst-sql-evidence.test.ts

/**
 * tests/analytics/analyst-sql-evidence.test.ts
 * =============================================
 * BI-P017 — the test that does NOT skip.
 *
 * The live suite (tests/integration/analyst-sql-clickhouse.integration.test.ts)
 * only runs when `CLICKHOUSE_URL` is set. On a runner without ClickHouse it
 * contributes nothing, and a suite that contributes nothing while reporting
 * green is how l0re's 28 Postgres tests stayed silently skipped for months.
 *
 * So the live run WRITES its result to
 * `evidence/bi-p017/analyst-sql-live-bypass.json`, that file is committed, and
 * this test — which always runs — checks it against the corpus:
 *
 *   * every attempt in the corpus must appear in the evidence, so adding an
 *     attack without re-running the live suite BREAKS THE BUILD rather than
 *     quietly going untested;
 *   * zero attempts may have leaked;
 *   * the legitimate statements must all have succeeded, so the evidence
 *     cannot be a recording of a database that refuses everything;
 *   * the counterfactual numbers must be present, so the evidence cannot be a
 *     recording of an EMPTY database, where "no foreign row came back" is
 *     true and meaningless.
 */
import fs from 'node:fs';
import path from 'node:path';
import { describe, expect, it } from 'vitest';
import { BYPASS_CORPUS, GUARD_ADMITS_ENGINE_REFUSES, LEGITIMATE_CORPUS } from './analyst-sql-bypass-corpus';

const EVIDENCE_PATH = path.join(process.cwd(), 'evidence/bi-p017/analyst-sql-live-bypass.json');

interface Evidence {
  clickhouse_version: string;
  recorded_at: string;
  base_table_rows_both_tenants: number;
  tenant_view_rows_alpha: number;
  attempts_total: number;
  attempts_that_leaked: number;
  legitimate_statements_that_succeeded: number;
  legitimate_statements_total: number;
  attempts: Array<{
    id: string;
    engine_http_status: number;
    engine_succeeded: boolean;
    leaked_foreign_tenant_row: boolean;
    leaked_ungranted_table_row: boolean;
    leaked_server_metadata: boolean;
    guard_verdict_for_reference: string;
  }>;
  legitimate: Array<{ id: string; ok: boolean; contains_beta_canary: boolean }>;
}

describe('BI-P017 — the committed live evidence covers the current corpus', () => {
  it('the evidence file exists', () => {
    expect(
      fs.existsSync(EVIDENCE_PATH),
      'evidence/bi-p017/analyst-sql-live-bypass.json is missing — run the live suite with CLICKHOUSE_URL set',
    ).toBe(true);
  });

  const evidence: Evidence = fs.existsSync(EVIDENCE_PATH)
    ? (JSON.parse(fs.readFileSync(EVIDENCE_PATH, 'utf8')) as Evidence)
    : ({ attempts: [], legitimate: [] } as unknown as Evidence);

  it('was recorded against a real ClickHouse, named by version', () => {
    expect(evidence.clickhouse_version).toMatch(/^\d+\.\d+/);
    expect(Date.parse(evidence.recorded_at)).toBeGreaterThan(0);
  });

  it("records the counterfactual, so 'nothing leaked' is not a statement about an empty database", () => {
    expect(evidence.base_table_rows_both_tenants).toBe(100);
    expect(evidence.tenant_view_rows_alpha).toBe(50);
  });

  it('covers EVERY attempt in the corpus — a new attack without a re-run fails here', () => {
    const recorded = new Set(evidence.attempts.map((a) => a.id));
    const missing = [...BYPASS_CORPUS, ...GUARD_ADMITS_ENGINE_REFUSES]
      .map((a) => a.id)
      .filter((id) => !recorded.has(id));
    expect(missing, `attempts never run against a live engine: ${missing.join(', ')}`).toEqual([]);
    expect(evidence.attempts_total).toBe(BYPASS_CORPUS.length + GUARD_ADMITS_ENGINE_REFUSES.length);
  });

  it('no attempt leaked a foreign-tenant row, an ungranted row, or a foreign object name', () => {
    const leaked = evidence.attempts.filter(
      (a) => a.leaked_foreign_tenant_row || a.leaked_ungranted_table_row || a.leaked_server_metadata,
    );
    expect(leaked.map((a) => a.id)).toEqual([]);
    expect(evidence.attempts_that_leaked).toBe(0);
  });

  it('records which attempts the ENGINE alone stopped, with the guard removed', () => {
    // The interesting number: attempts that the guard would have admitted and
    // the engine refused anyway. If it were zero, the layered claim would rest
    // entirely on the guard.
    const engineOnly = evidence.attempts.filter((a) => a.guard_verdict_for_reference === 'admitted');
    expect(engineOnly.length).toBeGreaterThan(0);
    for (const a of engineOnly) {
      expect(a.engine_succeeded, `${a.id} was admitted by the guard AND succeeded at the engine`).toBe(false);
    }
  });

  it('legitimate analyst work succeeded — the evidence is not a recording of a database that refuses everything', () => {
    expect(evidence.legitimate_statements_total).toBe(LEGITIMATE_CORPUS.length);
    expect(evidence.legitimate_statements_that_succeeded).toBe(LEGITIMATE_CORPUS.length);
    expect(evidence.legitimate.every((l) => l.contains_beta_canary === false)).toBe(true);
  });
});