/** * tests/analytics/bi-slo.test.ts * =============================== * BI-P103 — BI SLO, and the AUD0 usage events it is measured from. * * Two failure modes are attacked directly, because both look like a healthy * dashboard: * * 1. AN SLO REPORT OVER SILENCE. Zero events must not read as 100% * availability. Every objective in an empty window must come back * `insufficient_data`, and `fully_measured` must be false. * 2. A PRODUCER THAT LEAKS. The emitter is handed an execution record * carrying SQL text, a database error message quoting a personnummer, * filter VALUES, a tenant id and a result sample — everything AUD0's * BI-P076 ingest exists to refuse — and none of it may appear in any * emitted payload. The test greps the serialized events, and first * proves the grep finds those strings in the INPUT so a clean result * cannot be a broken detector. */ import { describe, expect, it } from 'vitest'; import { BI_SLO_CATALOGUE, evaluateBiSlos, percentile, summariseSloReport, type SloResult, } from '@/src/analytics/slo/bi-slo'; import { biUsageEventsForExecution, EMITTABLE_PAYLOAD_KEYS, type Aud0BiEvent, type ExecutionTelemetry, } from '@/src/analytics/slo/usage-events'; const FROM = new Date('2026-09-07T00:00:00.000Z'); const TO = new Date('2026-09-07T23:59:59.000Z'); function queryEvents(args: { n: number; surface: 'dashboard' | 'explore'; outcome: 'completed' | 'failed' | 'running'; durationMs: (i: number) => number; cacheHit?: boolean; idPrefix?: string; }): Aud0BiEvent[] { const out: Aud0BiEvent[] = []; for (let i = 0; i < args.n; i += 1) { const t: ExecutionTelemetry = { query_id: `${args.idPrefix ?? args.surface}_${i}`, surface: args.surface, state: args.outcome === 'completed' ? 'succeeded' : args.outcome === 'failed' ? 'failed' : 'running', started_at: new Date(FROM.getTime() + i * 1000).toISOString(), finished_at: new Date(FROM.getTime() + i * 1000 + args.durationMs(i)).toISOString(), duration_ms: args.durationMs(i), ...(args.cacheHit !== undefined ? { cache_hit: args.cacheHit } : {}), ...(args.outcome === 'failed' ? { error_code: 'TIMEOUT' } : {}), }; out.push(...biUsageEventsForExecution(t)); } return out; } function byId(results: SloResult[], id: string): SloResult { const r = results.find((x) => x.objective_id === id); if (!r) throw new Error(`no result for ${id}`); return r; } describe('BI-P103 — an SLO over silence is not a met SLO', () => { it('an empty window makes every measurable objective insufficient_data, never 100%', () => { const report = evaluateBiSlos({ events: [], from: FROM, to: TO }); expect(report.fully_measured).toBe(false); for (const r of report.results) { expect(['insufficient_data', 'not_measurable_here']).toContain(r.status); expect(r.status).not.toBe('met'); } const summary = summariseSloReport(report); expect(summary.met).toBe(0); expect(summary.measured_fraction).toBe(0); }); it('a started query that never finished is unresolved, not a success', () => { const events = queryEvents({ n: 200, surface: 'dashboard', outcome: 'running', durationMs: () => 100 }); const report = evaluateBiSlos({ events, from: FROM, to: TO }); expect(report.unresolved_queries).toBe(200); expect(byId(report.results, 'dash.availability').status).toBe('insufficient_data'); expect(byId(report.results, 'dash.availability').observed).toBeUndefined(); }); it('a percentile below min_samples is reported WITH its sample size and not judged', () => { const events = queryEvents({ n: 12, surface: 'explore', outcome: 'completed', durationMs: (i) => 1000 + i * 10, }); const p99 = byId(evaluateBiSlos({ events, from: FROM, to: TO }).results, 'explore.p99'); expect(p99.status).toBe('insufficient_data'); expect(p99.low_confidence).toBe(true); expect(p99.sample_size).toBe(12); expect(p99.observed).toBeDefined(); // reported… expect(p99.detail).toMatch(/not a percentile/); // …with the caveat attached }); it('freshness and delivery are declared not_measurable_here rather than proxied', () => { const report = evaluateBiSlos({ events: [], from: FROM, to: TO }); const freshness = byId(report.results, 'dash.freshness'); const delivery = byId(report.results, 'report.delivery_sla'); expect(freshness.status).toBe('not_measurable_here'); expect(freshness.detail).toMatch(/analytics_freshness_watermark/); expect(delivery.status).toBe('not_measurable_here'); expect(delivery.detail).toMatch(/n0tify_delivery_receipt/); }); }); describe('BI-P103 — real measurement from real events', () => { it('computes availability, error rate and both dashboard latencies', () => { const events = [ ...queryEvents({ n: 180, surface: 'dashboard', outcome: 'completed', cacheHit: false, durationMs: (i) => 500 + i * 10, idPrefix: 'cold', }), ...queryEvents({ n: 120, surface: 'dashboard', outcome: 'completed', cacheHit: true, durationMs: () => 120, idPrefix: 'warm', }), ...queryEvents({ n: 3, surface: 'dashboard', outcome: 'failed', durationMs: () => 900, idPrefix: 'bad', }), ]; const { results } = evaluateBiSlos({ events, from: FROM, to: TO }); const availability = byId(results, 'dash.availability'); expect(availability.sample_size).toBe(303); expect(availability.observed).toBeCloseTo(300 / 303, 6); expect(availability.status).toBe('breached'); // 0.9901 < 0.995 target const errorRate = byId(results, 'dash.error_rate'); expect(errorRate.observed).toBeCloseTo(3 / 303, 6); const cold = byId(results, 'dash.p95_first_query'); expect(cold.sample_size).toBe(183); // 180 cold + 3 failures, which had no cache hit expect(cold.status).toBe('met'); const cached = byId(results, 'dash.p95_cached'); expect(cached.sample_size).toBe(120); expect(cached.observed).toBe(120); expect(cached.status).toBe('met'); }); it('a breach is reported as a breach', () => { const events = queryEvents({ n: 60, surface: 'dashboard', outcome: 'completed', cacheHit: false, durationMs: () => 9000, }); expect( byId(evaluateBiSlos({ events, from: FROM, to: TO }).results, 'dash.p95_first_query'), ).toMatchObject({ status: 'breached', observed: 9000 }); }); it('events outside the window do not count', () => { const events = queryEvents({ n: 200, surface: 'dashboard', outcome: 'completed', durationMs: () => 100 }); const report = evaluateBiSlos({ events, from: new Date('2026-09-08T00:00:00.000Z'), to: new Date('2026-09-08T23:59:59.000Z'), }); expect(byId(report.results, 'dash.availability').status).toBe('insufficient_data'); }); it('percentile is nearest-rank — it never invents a value nobody experienced', () => { const sample = [10, 20, 30, 40]; expect(percentile(sample, 0.95)).toBe(40); expect(percentile(sample, 0.5)).toBe(20); expect(percentile([], 0.95)).toBeUndefined(); }); it('the catalogue covers all three workload classes and every field BI-P103 names', () => { const kinds = new Set(BI_SLO_CATALOGUE.map((o) => `${o.workload}:${o.kind}`)); for (const expected of [ 'interactive_dashboard:availability', 'interactive_dashboard:latency_p95_first', 'interactive_dashboard:latency_p95_cached', 'interactive_dashboard:error_rate', 'interactive_dashboard:freshness', 'exploration:latency_p95', 'exploration:latency_p99', 'exploration:max_result_rows', 'exploration:timeout', 'report:completion_sla', 'report:delivery_sla', ]) { expect(kinds.has(expected), expected).toBe(true); } }); }); describe('BI-P103 — the producer cannot leak a result into AUD0', () => { const hostile: ExecutionTelemetry = { query_id: 'q_1', compiled_query_hash: 'f'.repeat(64), compiled_sql: "SELECT * FROM customers WHERE personnummer = '19800101-1234'", error_message: 'duplicate key value violates unique constraint (personnummer)=(19800101-1234)', error_code: 'CONFLICT', tenant_id: 't_alpha', dataset_id: 'ds_sales', engine: 'clickhouse', surface: 'dashboard', state: 'failed', started_at: '2026-09-07T10:00:00.000Z', finished_at: '2026-09-07T10:00:02.000Z', duration_ms: 2000, filter_keys: ['region'], filter_values: { region: ['Gotland'] }, result_sample: [{ customer: 'Anna Andersson', revenue: 42 }], metric_name: 'revenue', dashboard_id: 'dash_sales', widget_count: 4, }; it('emits none of the forbidden material', () => { const events = biUsageEventsForExecution(hostile); const serialized = JSON.stringify(events); // The detector must be able to find these strings — they are in the input. const input = JSON.stringify(hostile); for (const secret of ['19800101-1234', 'Anna Andersson', 'Gotland', 't_alpha', 'SELECT *']) { expect(input, `detector check for ${secret}`).toContain(secret); } for (const secret of [ '19800101-1234', 'Anna Andersson', 'Gotland', 't_alpha', 'SELECT *', 'duplicate key', ]) { expect(serialized, secret).not.toContain(secret); } }); it('every emitted key is on the allowlist AUD0 accepts', () => { const events = biUsageEventsForExecution(hostile); for (const e of events) { for (const key of Object.keys(e.payload)) { expect(EMITTABLE_PAYLOAD_KEYS, `${e.event_type}.${key}`).toContain(key); } } }); it('a query_hash that is not a sha256 digest is dropped, not reshaped', () => { const events = biUsageEventsForExecution({ ...hostile, compiled_query_hash: 'SELECT 1' }); for (const e of events) expect(e.payload.query_hash).toBeUndefined(); }); it('a running query emits only "started" — it is neither a success nor a failure', () => { const events = biUsageEventsForExecution({ ...hostile, state: 'running', metric_name: undefined, dashboard_id: undefined, }); expect(events.map((e) => e.event_type)).toEqual(['aud0.analytics_query_started']); }); it('a cancelled query is reported as failed with a CODE', () => { const events = biUsageEventsForExecution({ ...hostile, state: 'cancelled', error_code: undefined, metric_name: undefined, dashboard_id: undefined, }); const failed = events.find((e) => e.event_type === 'aud0.analytics_query_failed'); expect(failed?.payload.error_code).toBe('CANCELLED'); expect(failed?.payload.error_message).toBeUndefined(); }); });