0pt/tests/analytics/bi-slo.test.ts

/**
 * tests/analytics/bi-slo.test.ts
 * ===============================
 * BI-P103 — BI SLO, and the AUD0 usage events it is measured from.
 *
 * Two failure modes are attacked directly, because both look like a healthy
 * dashboard:
 *
 *   1. AN SLO REPORT OVER SILENCE. Zero events must not read as 100%
 *      availability. Every objective in an empty window must come back
 *      `insufficient_data`, and `fully_measured` must be false.
 *   2. A PRODUCER THAT LEAKS. The emitter is handed an execution record
 *      carrying SQL text, a database error message quoting a personnummer,
 *      filter VALUES, a tenant id and a result sample — everything AUD0's
 *      BI-P076 ingest exists to refuse — and none of it may appear in any
 *      emitted payload. The test greps the serialized events, and first
 *      proves the grep finds those strings in the INPUT so a clean result
 *      cannot be a broken detector.
 */
import { describe, expect, it } from 'vitest';
import {
  BI_SLO_CATALOGUE,
  evaluateBiSlos,
  percentile,
  summariseSloReport,
  type SloResult,
} from '@/src/analytics/slo/bi-slo';
import {
  biUsageEventsForExecution,
  EMITTABLE_PAYLOAD_KEYS,
  type Aud0BiEvent,
  type ExecutionTelemetry,
} from '@/src/analytics/slo/usage-events';

const FROM = new Date('2026-09-07T00:00:00.000Z');
const TO = new Date('2026-09-07T23:59:59.000Z');

function queryEvents(args: {
  n: number;
  surface: 'dashboard' | 'explore';
  outcome: 'completed' | 'failed' | 'running';
  durationMs: (i: number) => number;
  cacheHit?: boolean;
  idPrefix?: string;
}): Aud0BiEvent[] {
  const out: Aud0BiEvent[] = [];
  for (let i = 0; i < args.n; i += 1) {
    const t: ExecutionTelemetry = {
      query_id: `${args.idPrefix ?? args.surface}_${i}`,
      surface: args.surface,
      state: args.outcome === 'completed' ? 'succeeded' : args.outcome === 'failed' ? 'failed' : 'running',
      started_at: new Date(FROM.getTime() + i * 1000).toISOString(),
      finished_at: new Date(FROM.getTime() + i * 1000 + args.durationMs(i)).toISOString(),
      duration_ms: args.durationMs(i),
      ...(args.cacheHit !== undefined ? { cache_hit: args.cacheHit } : {}),
      ...(args.outcome === 'failed' ? { error_code: 'TIMEOUT' } : {}),
    };
    out.push(...biUsageEventsForExecution(t));
  }
  return out;
}

function byId(results: SloResult[], id: string): SloResult {
  const r = results.find((x) => x.objective_id === id);
  if (!r) throw new Error(`no result for ${id}`);
  return r;
}

describe('BI-P103 — an SLO over silence is not a met SLO', () => {
  it('an empty window makes every measurable objective insufficient_data, never 100%', () => {
    const report = evaluateBiSlos({ events: [], from: FROM, to: TO });
    expect(report.fully_measured).toBe(false);
    for (const r of report.results) {
      expect(['insufficient_data', 'not_measurable_here']).toContain(r.status);
      expect(r.status).not.toBe('met');
    }
    const summary = summariseSloReport(report);
    expect(summary.met).toBe(0);
    expect(summary.measured_fraction).toBe(0);
  });

  it('a started query that never finished is unresolved, not a success', () => {
    const events = queryEvents({ n: 200, surface: 'dashboard', outcome: 'running', durationMs: () => 100 });
    const report = evaluateBiSlos({ events, from: FROM, to: TO });
    expect(report.unresolved_queries).toBe(200);
    expect(byId(report.results, 'dash.availability').status).toBe('insufficient_data');
    expect(byId(report.results, 'dash.availability').observed).toBeUndefined();
  });

  it('a percentile below min_samples is reported WITH its sample size and not judged', () => {
    const events = queryEvents({
      n: 12,
      surface: 'explore',
      outcome: 'completed',
      durationMs: (i) => 1000 + i * 10,
    });
    const p99 = byId(evaluateBiSlos({ events, from: FROM, to: TO }).results, 'explore.p99');
    expect(p99.status).toBe('insufficient_data');
    expect(p99.low_confidence).toBe(true);
    expect(p99.sample_size).toBe(12);
    expect(p99.observed).toBeDefined(); // reported…
    expect(p99.detail).toMatch(/not a percentile/); // …with the caveat attached
  });

  it('freshness and delivery are declared not_measurable_here rather than proxied', () => {
    const report = evaluateBiSlos({ events: [], from: FROM, to: TO });
    const freshness = byId(report.results, 'dash.freshness');
    const delivery = byId(report.results, 'report.delivery_sla');
    expect(freshness.status).toBe('not_measurable_here');
    expect(freshness.detail).toMatch(/analytics_freshness_watermark/);
    expect(delivery.status).toBe('not_measurable_here');
    expect(delivery.detail).toMatch(/n0tify_delivery_receipt/);
  });
});

describe('BI-P103 — real measurement from real events', () => {
  it('computes availability, error rate and both dashboard latencies', () => {
    const events = [
      ...queryEvents({
        n: 180,
        surface: 'dashboard',
        outcome: 'completed',
        cacheHit: false,
        durationMs: (i) => 500 + i * 10,
        idPrefix: 'cold',
      }),
      ...queryEvents({
        n: 120,
        surface: 'dashboard',
        outcome: 'completed',
        cacheHit: true,
        durationMs: () => 120,
        idPrefix: 'warm',
      }),
      ...queryEvents({
        n: 3,
        surface: 'dashboard',
        outcome: 'failed',
        durationMs: () => 900,
        idPrefix: 'bad',
      }),
    ];
    const { results } = evaluateBiSlos({ events, from: FROM, to: TO });

    const availability = byId(results, 'dash.availability');
    expect(availability.sample_size).toBe(303);
    expect(availability.observed).toBeCloseTo(300 / 303, 6);
    expect(availability.status).toBe('breached'); // 0.9901 < 0.995 target

    const errorRate = byId(results, 'dash.error_rate');
    expect(errorRate.observed).toBeCloseTo(3 / 303, 6);

    const cold = byId(results, 'dash.p95_first_query');
    expect(cold.sample_size).toBe(183); // 180 cold + 3 failures, which had no cache hit
    expect(cold.status).toBe('met');

    const cached = byId(results, 'dash.p95_cached');
    expect(cached.sample_size).toBe(120);
    expect(cached.observed).toBe(120);
    expect(cached.status).toBe('met');
  });

  it('a breach is reported as a breach', () => {
    const events = queryEvents({
      n: 60,
      surface: 'dashboard',
      outcome: 'completed',
      cacheHit: false,
      durationMs: () => 9000,
    });
    expect(
      byId(evaluateBiSlos({ events, from: FROM, to: TO }).results, 'dash.p95_first_query'),
    ).toMatchObject({ status: 'breached', observed: 9000 });
  });

  it('events outside the window do not count', () => {
    const events = queryEvents({ n: 200, surface: 'dashboard', outcome: 'completed', durationMs: () => 100 });
    const report = evaluateBiSlos({
      events,
      from: new Date('2026-09-08T00:00:00.000Z'),
      to: new Date('2026-09-08T23:59:59.000Z'),
    });
    expect(byId(report.results, 'dash.availability').status).toBe('insufficient_data');
  });

  it('percentile is nearest-rank — it never invents a value nobody experienced', () => {
    const sample = [10, 20, 30, 40];
    expect(percentile(sample, 0.95)).toBe(40);
    expect(percentile(sample, 0.5)).toBe(20);
    expect(percentile([], 0.95)).toBeUndefined();
  });

  it('the catalogue covers all three workload classes and every field BI-P103 names', () => {
    const kinds = new Set(BI_SLO_CATALOGUE.map((o) => `${o.workload}:${o.kind}`));
    for (const expected of [
      'interactive_dashboard:availability',
      'interactive_dashboard:latency_p95_first',
      'interactive_dashboard:latency_p95_cached',
      'interactive_dashboard:error_rate',
      'interactive_dashboard:freshness',
      'exploration:latency_p95',
      'exploration:latency_p99',
      'exploration:max_result_rows',
      'exploration:timeout',
      'report:completion_sla',
      'report:delivery_sla',
    ]) {
      expect(kinds.has(expected), expected).toBe(true);
    }
  });
});

describe('BI-P103 — the producer cannot leak a result into AUD0', () => {
  const hostile: ExecutionTelemetry = {
    query_id: 'q_1',
    compiled_query_hash: 'f'.repeat(64),
    compiled_sql: "SELECT * FROM customers WHERE personnummer = '19800101-1234'",
    error_message: 'duplicate key value violates unique constraint (personnummer)=(19800101-1234)',
    error_code: 'CONFLICT',
    tenant_id: 't_alpha',
    dataset_id: 'ds_sales',
    engine: 'clickhouse',
    surface: 'dashboard',
    state: 'failed',
    started_at: '2026-09-07T10:00:00.000Z',
    finished_at: '2026-09-07T10:00:02.000Z',
    duration_ms: 2000,
    filter_keys: ['region'],
    filter_values: { region: ['Gotland'] },
    result_sample: [{ customer: 'Anna Andersson', revenue: 42 }],
    metric_name: 'revenue',
    dashboard_id: 'dash_sales',
    widget_count: 4,
  };

  it('emits none of the forbidden material', () => {
    const events = biUsageEventsForExecution(hostile);
    const serialized = JSON.stringify(events);

    // The detector must be able to find these strings — they are in the input.
    const input = JSON.stringify(hostile);
    for (const secret of ['19800101-1234', 'Anna Andersson', 'Gotland', 't_alpha', 'SELECT *']) {
      expect(input, `detector check for ${secret}`).toContain(secret);
    }
    for (const secret of [
      '19800101-1234',
      'Anna Andersson',
      'Gotland',
      't_alpha',
      'SELECT *',
      'duplicate key',
    ]) {
      expect(serialized, secret).not.toContain(secret);
    }
  });

  it('every emitted key is on the allowlist AUD0 accepts', () => {
    const events = biUsageEventsForExecution(hostile);
    for (const e of events) {
      for (const key of Object.keys(e.payload)) {
        expect(EMITTABLE_PAYLOAD_KEYS, `${e.event_type}.${key}`).toContain(key);
      }
    }
  });

  it('a query_hash that is not a sha256 digest is dropped, not reshaped', () => {
    const events = biUsageEventsForExecution({ ...hostile, compiled_query_hash: 'SELECT 1' });
    for (const e of events) expect(e.payload.query_hash).toBeUndefined();
  });

  it('a running query emits only "started" — it is neither a success nor a failure', () => {
    const events = biUsageEventsForExecution({
      ...hostile,
      state: 'running',
      metric_name: undefined,
      dashboard_id: undefined,
    });
    expect(events.map((e) => e.event_type)).toEqual(['aud0.analytics_query_started']);
  });

  it('a cancelled query is reported as failed with a CODE', () => {
    const events = biUsageEventsForExecution({
      ...hostile,
      state: 'cancelled',
      error_code: undefined,
      metric_name: undefined,
      dashboard_id: undefined,
    });
    const failed = events.find((e) => e.event_type === 'aud0.analytics_query_failed');
    expect(failed?.payload.error_code).toBe('CANCELLED');
    expect(failed?.payload.error_message).toBeUndefined();
  });
});