diff --git a/src/analytics/slo/bi-slo.ts b/src/analytics/slo/bi-slo.ts new file mode 100644 index 0000000..b4bad4b --- /dev/null +++ b/src/analytics/slo/bi-slo.ts @@ -0,0 +1,501 @@ +/** + * src/analytics/slo/bi-slo.ts + * ============================ + * BI-P103 — BI SLO. DoD: "measured baselines exist". + * + * An SLO you cannot measure from real telemetry is a wish, so every objective + * here declares WHERE its number comes from, and the evaluator refuses to + * produce a number it does not have. Three ways that refusal shows up, each of + * which is a lie the usual implementation tells: + * + * insufficient_data nothing (or too little) was observed in the window. + * An availability objective evaluated over zero events + * is NOT 100% attainment — it is silence. This is the + * single most common false green in SLO reporting, and + * it is the same shape BI-P025 found in a quality + * check over an empty table. + * not_measurable_here the objective is real but these events cannot + * measure it. Dashboard data FRESHNESS is a property + * of the dataset (BI-P026 / BI-P105 watermarks), not + * of a usage event, and REPORT DELIVERY is N0TIFY's + * fact — aud0's seven types carry no delivery event. + * Saying so beats fabricating a proxy. + * low_confidence a p95 or p99 computed from a handful of samples. The + * number is reported WITH its sample size and marked, + * because a p99 over 12 requests is the 12th-slowest + * request wearing a percentile's clothes. + * + * The measurement path is aud0's seven BI usage event types (BI-P076, merged + * in aud0#34), produced by src/analytics/slo/usage-events.ts. There is + * deliberately no second telemetry path: two measurement systems disagree, and + * the disagreement is discovered during an incident. + */ +import type { Aud0BiEvent } from './usage-events'; + +export const BI_WORKLOAD_CLASSES = ['interactive_dashboard', 'exploration', 'report'] as const; +export type BiWorkloadClass = (typeof BI_WORKLOAD_CLASSES)[number]; + +export const SLO_OBJECTIVE_KINDS = [ + 'availability', + 'latency_p95_first', + 'latency_p95_cached', + 'latency_p95', + 'latency_p99', + 'error_rate', + 'freshness', + 'max_result_rows', + 'timeout', + 'completion_sla', + 'delivery_sla', +] as const; +export type SloObjectiveKind = (typeof SLO_OBJECTIVE_KINDS)[number]; + +/** Where the number comes from. `none` is an admission, not a placeholder. */ +export const MEASUREMENT_SOURCES = [ + 'aud0_bi_usage_events', + 'analytics_freshness_watermark', + 'n0tify_delivery_receipt', + 'engine_configuration', +] as const; +export type MeasurementSource = (typeof MEASUREMENT_SOURCES)[number]; + +export interface SloObjective { + objective_id: string; + workload: BiWorkloadClass; + kind: SloObjectiveKind; + /** Target value; unit is fixed by kind (ms, ratio, rows, seconds). */ + target: number; + unit: 'ms' | 'ratio' | 'rows' | 'seconds'; + /** Higher-is-better (availability) vs lower-is-better (latency). */ + direction: 'at_least' | 'at_most'; + measured_from: MeasurementSource; + /** Fewer samples than this and the verdict carries low_confidence. */ + min_samples: number; + rationale: string; +} + +/** + * The catalogue, field-for-field with what BI-P103 lists per workload class. + * Targets are STARTING POINTS chosen to be honest about what has been + * measured: they are the thresholds the evaluator compares against, and + * evidence/bi-p103 records the first baselines rather than pretending the + * targets were derived from a year of production traffic. + */ +export const BI_SLO_CATALOGUE: readonly SloObjective[] = [ + // ── interactive dashboard ──────────────────────────────────────────────── + { + objective_id: 'dash.availability', + workload: 'interactive_dashboard', + kind: 'availability', + target: 0.995, + unit: 'ratio', + direction: 'at_least', + measured_from: 'aud0_bi_usage_events', + min_samples: 100, + rationale: 'completed / (completed + failed) for dashboard-surface queries.', + }, + { + objective_id: 'dash.p95_first_query', + workload: 'interactive_dashboard', + kind: 'latency_p95_first', + target: 3000, + unit: 'ms', + direction: 'at_most', + measured_from: 'aud0_bi_usage_events', + min_samples: 50, + rationale: 'cache_hit=false: what a user waits for when they open the dashboard cold.', + }, + { + objective_id: 'dash.p95_cached', + workload: 'interactive_dashboard', + kind: 'latency_p95_cached', + target: 400, + unit: 'ms', + direction: 'at_most', + measured_from: 'aud0_bi_usage_events', + min_samples: 50, + rationale: + 'cache_hit=true: BI-P020 cache hits, measured separately so a high hit rate cannot hide a slow cold path.', + }, + { + objective_id: 'dash.error_rate', + workload: 'interactive_dashboard', + kind: 'error_rate', + target: 0.005, + unit: 'ratio', + direction: 'at_most', + measured_from: 'aud0_bi_usage_events', + min_samples: 100, + rationale: 'failed / terminal. Cancellations count as failures here: the user did not get an answer.', + }, + { + objective_id: 'dash.freshness', + workload: 'interactive_dashboard', + kind: 'freshness', + target: 900, + unit: 'seconds', + direction: 'at_most', + measured_from: 'analytics_freshness_watermark', + min_samples: 1, + rationale: + 'Data age is a property of the DATASET (BI-P026 freshness, BI-P105 watermark), not of a usage ' + + 'event. Usage events cannot measure it and this evaluator does not pretend to.', + }, + // ── exploration ────────────────────────────────────────────────────────── + { + objective_id: 'explore.p95', + workload: 'exploration', + kind: 'latency_p95', + target: 10_000, + unit: 'ms', + direction: 'at_most', + measured_from: 'aud0_bi_usage_events', + min_samples: 50, + rationale: 'explore-surface queries; an analyst tolerates more than a dashboard viewer.', + }, + { + objective_id: 'explore.p99', + workload: 'exploration', + kind: 'latency_p99', + target: 60_000, + unit: 'ms', + direction: 'at_most', + measured_from: 'aud0_bi_usage_events', + min_samples: 200, + rationale: 'p99 needs a genuinely large sample; below min_samples it is reported as low confidence.', + }, + { + objective_id: 'explore.max_result', + workload: 'exploration', + kind: 'max_result_rows', + target: 1_000_000, + unit: 'rows', + direction: 'at_most', + measured_from: 'aud0_bi_usage_events', + min_samples: 1, + rationale: 'BI-P012 truncates at the engine ceiling; this watches what actually came back.', + }, + { + objective_id: 'explore.timeout', + workload: 'exploration', + kind: 'timeout', + target: 300, + unit: 'seconds', + direction: 'at_most', + measured_from: 'engine_configuration', + min_samples: 1, + rationale: 'The configured ceiling (BI-P012 settings profile), not an observation.', + }, + // ── reports ────────────────────────────────────────────────────────────── + { + objective_id: 'report.completion_sla', + workload: 'report', + kind: 'completion_sla', + target: 900_000, + unit: 'ms', + direction: 'at_most', + measured_from: 'aud0_bi_usage_events', + min_samples: 20, + rationale: 'aud0.report_generated duration_ms: submitted to produced.', + }, + { + objective_id: 'report.delivery_sla', + workload: 'report', + kind: 'delivery_sla', + target: 300_000, + unit: 'ms', + direction: 'at_most', + measured_from: 'n0tify_delivery_receipt', + min_samples: 20, + rationale: + "Produced is not delivered. aud0 has no delivery event; the receipt is N0TIFY's, so this " + + 'objective is declared and reported unmeasured rather than proxied by completion.', + }, +]; + +export const SLO_STATUSES = ['met', 'breached', 'insufficient_data', 'not_measurable_here'] as const; +export type SloStatus = (typeof SLO_STATUSES)[number]; + +export interface SloResult { + objective_id: string; + workload: BiWorkloadClass; + kind: SloObjectiveKind; + status: SloStatus; + target: number; + unit: SloObjective['unit']; + /** The measured value, or undefined when nothing could be measured. */ + observed?: number; + sample_size: number; + /** True when a percentile was computed from fewer samples than min_samples. */ + low_confidence?: boolean; + detail: string; +} + +export interface SloReport { + window_from: string; + window_to: string; + results: SloResult[]; + /** Queries that started and never reached a terminal event in the window. */ + unresolved_queries: number; + /** True only when EVERY objective produced a real number. */ + fully_measured: boolean; +} + +interface QueryFact { + query_id: string; + surface?: string; + started_at: number; + outcome?: 'completed' | 'failed'; + duration_ms?: number; + row_count?: number; + cache_hit?: boolean; +} + +function num(v: unknown): number | undefined { + return typeof v === 'number' && Number.isFinite(v) ? v : undefined; +} + +function str(v: unknown): string | undefined { + return typeof v === 'string' ? v : undefined; +} + +/** + * Nearest-rank percentile. No interpolation: with small samples an + * interpolated p99 invents a value that no request ever experienced. + */ +export function percentile(sorted: number[], q: number): number | undefined { + if (sorted.length === 0) return undefined; + const rank = Math.ceil(q * sorted.length); + return sorted[Math.min(sorted.length - 1, Math.max(0, rank - 1))]; +} + +/** Fold the seven event types into per-query facts. */ +function foldQueries(events: Aud0BiEvent[]): Map { + const facts = new Map(); + for (const e of events) { + const qid = str(e.payload.query_id); + if (!qid) continue; + const at = Date.parse(e.occurred_at); + const existing = facts.get(qid) ?? { query_id: qid, started_at: Number.isNaN(at) ? 0 : at }; + if (e.payload.surface !== undefined) existing.surface = str(e.payload.surface); + if (e.event_type === 'aud0.analytics_query_started' && !Number.isNaN(at)) existing.started_at = at; + if (e.event_type === 'aud0.analytics_query_completed') { + existing.outcome = 'completed'; + existing.duration_ms = num(e.payload.duration_ms); + existing.row_count = num(e.payload.row_count); + existing.cache_hit = e.payload.cache_hit === true; + } + if (e.event_type === 'aud0.analytics_query_failed') { + existing.outcome = 'failed'; + existing.duration_ms = num(e.payload.duration_ms); + } + facts.set(qid, existing); + } + return facts; +} + +function result( + o: SloObjective, + status: SloStatus, + sample_size: number, + detail: string, + observed?: number, + low_confidence?: boolean, +): SloResult { + const r: SloResult = { + objective_id: o.objective_id, + workload: o.workload, + kind: o.kind, + status, + target: o.target, + unit: o.unit, + sample_size, + detail, + }; + if (observed !== undefined) r.observed = observed; + if (low_confidence) r.low_confidence = true; + return r; +} + +function compare(o: SloObjective, observed: number): SloStatus { + return o.direction === 'at_least' + ? observed >= o.target + ? 'met' + : 'breached' + : observed <= o.target + ? 'met' + : 'breached'; +} + +/** + * Evaluate the catalogue over a window of AUD0 BI usage events. + * + * Events are DATA: they arrive from aud0's read model, they were written by + * other services, and nothing in them is treated as an instruction or as a + * schema. Unknown event types and unknown payload keys are ignored, never + * spread. + */ +export function evaluateBiSlos(args: { + events: Aud0BiEvent[]; + from: Date; + to: Date; + objectives?: readonly SloObjective[]; +}): SloReport { + const objectives = args.objectives ?? BI_SLO_CATALOGUE; + const fromMs = args.from.getTime(); + const toMs = args.to.getTime(); + const inWindow = args.events.filter((e) => { + const t = Date.parse(e.occurred_at); + return !Number.isNaN(t) && t >= fromMs && t <= toMs; + }); + + const queries = [...foldQueries(inWindow).values()]; + const dashboardQueries = queries.filter((q) => q.surface === 'dashboard'); + const exploreQueries = queries.filter((q) => q.surface === 'explore'); + const terminal = (qs: QueryFact[]) => qs.filter((q) => q.outcome !== undefined); + const durations = (qs: QueryFact[]) => + qs + .map((q) => q.duration_ms) + .filter((d): d is number => d !== undefined) + .sort((a, b) => a - b); + + const reportDurations = inWindow + .filter((e) => e.event_type === 'aud0.report_generated') + .map((e) => num(e.payload.duration_ms)) + .filter((d): d is number => d !== undefined) + .sort((a, b) => a - b); + + const results = objectives.map((o) => { + if (o.measured_from !== 'aud0_bi_usage_events') { + return result( + o, + 'not_measurable_here', + 0, + `measured from ${o.measured_from}; BI usage events cannot answer it — ${o.rationale}`, + ); + } + + switch (o.kind) { + case 'availability': + case 'error_rate': { + const t = terminal(dashboardQueries); + if (t.length === 0) { + return result(o, 'insufficient_data', 0, 'no terminal dashboard query events in the window'); + } + const failed = t.filter((q) => q.outcome === 'failed').length; + const observed = o.kind === 'availability' ? (t.length - failed) / t.length : failed / t.length; + const status = compare(o, observed); + return result( + o, + t.length < o.min_samples ? 'insufficient_data' : status, + t.length, + t.length < o.min_samples + ? `${t.length} terminal queries is below the ${o.min_samples} this objective requires; ` + + `the ratio ${observed.toFixed(4)} is reported but not judged` + : `${failed} failed of ${t.length} terminal queries`, + observed, + t.length < o.min_samples, + ); + } + case 'latency_p95_first': + case 'latency_p95_cached': { + const wanted = o.kind === 'latency_p95_cached'; + const sample = durations(dashboardQueries.filter((q) => (q.cache_hit === true) === wanted)); + const p95 = percentile(sample, 0.95); + if (p95 === undefined) { + return result( + o, + 'insufficient_data', + 0, + `no ${wanted ? 'cached' : 'uncached'} dashboard durations`, + ); + } + const low = sample.length < o.min_samples; + return result( + o, + low ? 'insufficient_data' : compare(o, p95), + sample.length, + low + ? `p95 over ${sample.length} samples (needs ${o.min_samples}) — reported, not judged` + : `p95 over ${sample.length} samples`, + p95, + low, + ); + } + case 'latency_p95': + case 'latency_p99': { + const sample = durations(exploreQueries); + const q = o.kind === 'latency_p99' ? 0.99 : 0.95; + const value = percentile(sample, q); + if (value === undefined) return result(o, 'insufficient_data', 0, 'no exploration durations'); + const low = sample.length < o.min_samples; + return result( + o, + low ? 'insufficient_data' : compare(o, value), + sample.length, + low + ? `p${q * 100} over ${sample.length} samples (needs ${o.min_samples}); with this few samples ` + + `it is the ${Math.ceil((1 - q) * sample.length) || 1}-slowest request, not a percentile` + : `p${q * 100} over ${sample.length} samples`, + value, + low, + ); + } + case 'max_result_rows': { + const rows = exploreQueries + .map((q) => q.row_count) + .filter((r): r is number => r !== undefined) + .sort((a, b) => b - a); + if (rows.length === 0) return result(o, 'insufficient_data', 0, 'no exploration row counts'); + return result(o, compare(o, rows[0]), rows.length, `largest result seen in the window`, rows[0]); + } + case 'completion_sla': { + const p95 = percentile(reportDurations, 0.95); + if (p95 === undefined) return result(o, 'insufficient_data', 0, 'no report_generated events'); + const low = reportDurations.length < o.min_samples; + return result( + o, + low ? 'insufficient_data' : compare(o, p95), + reportDurations.length, + low ? `p95 over ${reportDurations.length} reports (needs ${o.min_samples})` : 'p95 report duration', + p95, + low, + ); + } + default: + return result(o, 'not_measurable_here', 0, `no evaluator for kind "${o.kind}"`); + } + }); + + const unresolved = queries.filter((q) => q.outcome === undefined).length; + return { + window_from: args.from.toISOString(), + window_to: args.to.toISOString(), + results, + unresolved_queries: unresolved, + fully_measured: results.every((r) => r.status === 'met' || r.status === 'breached'), + }; +} + +/** + * One line an operator can read: how many objectives are actually being + * measured. "9 of 11 met" is meaningless if the other two were never + * measured, so the summary counts them separately and never rolls them in. + */ +export function summariseSloReport(report: SloReport): { + met: number; + breached: number; + insufficient_data: number; + not_measurable_here: number; + measured_fraction: number; +} { + const count = (s: SloStatus) => report.results.filter((r) => r.status === s).length; + const measured = count('met') + count('breached'); + return { + met: count('met'), + breached: count('breached'), + insufficient_data: count('insufficient_data'), + not_measurable_here: count('not_measurable_here'), + measured_fraction: report.results.length === 0 ? 0 : measured / report.results.length, + }; +} diff --git a/src/analytics/slo/usage-events.ts b/src/analytics/slo/usage-events.ts new file mode 100644 index 0000000..dec22ba --- /dev/null +++ b/src/analytics/slo/usage-events.ts @@ -0,0 +1,209 @@ +/** + * src/analytics/slo/usage-events.ts + * ================================== + * BI-P103's measurement path: 0pt emits the AUD0 BI usage events (aud0#34, + * BI-P076) that the SLO evaluator then reads. + * + * aud0 says it plainly in docs/BI_ANALYTICS_USAGE.md: "Zero real producers + * emit these seven types today." An SLO computed from a second, private + * telemetry path would be a number nobody else can audit, so 0pt becomes a + * producer instead of inventing one. + * + * THE PAYLOAD IS AN ALLOWLIST, BUILT FIELD BY NAMED FIELD. AUD0's ingest + * refuses result-bearing payloads with a 400 and its schemas are `.strict()`, + * so a leak here is a 400 rather than a breach — but a producer that has to be + * refused is a producer that is broken. `biUsageEventsForExecution` therefore + * never spreads an execution record; it names each field, and + * tests/analytics/bi-slo.test.ts feeds it a record carrying SQL text, an error + * message, a tenant id and a result sample, then asserts none of it appears in + * the output. + * + * WHAT IS DELIBERATELY NOT SENT: `compiled_query_hash` goes out as + * `query_hash` (aud0 requires a 64-hex digest and nothing else), the statement + * never leaves 0pt at all, `error_code` travels but `error_message` does not, + * and `row_count` travels but no row does. + */ + +/** The seven event types, named exactly as aud0 registers them. */ +export const AUD0_BI_EVENT_TYPES = [ + 'aud0.analytics_query_started', + 'aud0.analytics_query_completed', + 'aud0.analytics_query_failed', + 'aud0.dashboard_viewed', + 'aud0.report_generated', + 'aud0.dataset_exported', + 'aud0.metric_queried', +] as const; +export type Aud0BiEventType = (typeof AUD0_BI_EVENT_TYPES)[number]; + +/** aud0's closed surface list (BI_SURFACES). */ +export const BI_SURFACES = ['dashboard', 'report', 'explore', 'api', 'scheduled', 'notebook'] as const; +export type BiSurface = (typeof BI_SURFACES)[number]; + +export interface Aud0BiEvent { + event_type: Aud0BiEventType; + /** Bounded, allowlisted metadata. Never a result, a statement or a message. */ + payload: Record; + /** Occurrence time, used by the SLO window. */ + occurred_at: string; +} + +/** + * The 0pt-side view of one execution. Intentionally a SUPERSET of what may be + * emitted — it carries the fields 0pt genuinely has, including the ones that + * must never leave — so that the mapping below is a real filter rather than a + * rename. + */ +export interface ExecutionTelemetry { + query_id: string; + /** 64-hex; anything else is dropped rather than reshaped. */ + compiled_query_hash?: string; + /** NEVER emitted. Present here because 0pt has it. */ + compiled_sql?: string; + /** NEVER emitted. */ + error_message?: string; + error_code?: string; + /** NEVER emitted — a consumer must not be able to name a tenant, BI-P016. */ + tenant_id?: string; + dataset_id?: string; + engine?: string; + surface?: BiSurface; + state: 'succeeded' | 'failed' | 'cancelled' | 'running'; + started_at: string; + finished_at?: string; + duration_ms?: number; + row_count?: number; + bytes_scanned?: number; + cache_hit?: boolean; + filter_keys?: string[]; + /** NEVER emitted: filter VALUES are facts about a subject. */ + filter_values?: Record; + /** NEVER emitted. */ + result_sample?: unknown; + metric_name?: string; + dimension_keys?: string[]; + granularity?: string; + dashboard_id?: string; + widget_count?: number; +} + +const SHA256 = /^[0-9a-f]{64}$/; +const MAX_KEY_NAMES = 50; +const MAX_SHORT_TEXT = 120; + +function shortText(v: string | undefined): string | undefined { + if (!v) return undefined; + return v.slice(0, MAX_SHORT_TEXT); +} + +function keyNames(v: string[] | undefined): string[] | undefined { + if (!v || v.length === 0) return undefined; + return v.slice(0, MAX_KEY_NAMES).map((k) => k.slice(0, MAX_SHORT_TEXT)); +} + +function put( + payload: Record, + key: string, + value: string | number | boolean | string[] | undefined, +): void { + if (value === undefined) return; + payload[key] = value; +} + +/** + * Map one execution to its AUD0 usage events. + * + * A running execution emits only `started`: a query that has not finished is + * neither a success nor a failure, and the SLO evaluator counts it as + * unresolved rather than quietly as one or the other. + */ +export function biUsageEventsForExecution(t: ExecutionTelemetry): Aud0BiEvent[] { + const base: Record = {}; + put(base, 'query_id', t.query_id); + if (t.compiled_query_hash && SHA256.test(t.compiled_query_hash)) { + put(base, 'query_hash', t.compiled_query_hash); + } + put(base, 'dataset_id', t.dataset_id); + put(base, 'engine', shortText(t.engine)); + put(base, 'surface', t.surface); + put(base, 'filter_keys', keyNames(t.filter_keys)); + + const events: Aud0BiEvent[] = [ + { event_type: 'aud0.analytics_query_started', payload: { ...base }, occurred_at: t.started_at }, + ]; + + if (t.state === 'succeeded') { + const payload = { ...base }; + put(payload, 'duration_ms', t.duration_ms); + put(payload, 'row_count', t.row_count); + put(payload, 'bytes_scanned', t.bytes_scanned); + put(payload, 'cache_hit', t.cache_hit); + events.push({ + event_type: 'aud0.analytics_query_completed', + payload, + occurred_at: t.finished_at ?? t.started_at, + }); + } else if (t.state === 'failed' || t.state === 'cancelled') { + const payload = { ...base }; + put(payload, 'duration_ms', t.duration_ms); + // A CODE. `error_message` is never mapped, at any length. + put( + payload, + 'error_code', + shortText(t.error_code ?? (t.state === 'cancelled' ? 'CANCELLED' : undefined)), + ); + events.push({ + event_type: 'aud0.analytics_query_failed', + payload, + occurred_at: t.finished_at ?? t.started_at, + }); + } + + if (t.metric_name) { + const payload: Record = {}; + put(payload, 'metric_name', shortText(t.metric_name)); + put(payload, 'dataset_id', t.dataset_id); + put(payload, 'surface', t.surface); + put(payload, 'engine', shortText(t.engine)); + put(payload, 'dimension_keys', keyNames(t.dimension_keys)); + put(payload, 'granularity', shortText(t.granularity)); + put(payload, 'row_count', t.row_count); + put(payload, 'duration_ms', t.duration_ms); + events.push({ + event_type: 'aud0.metric_queried', + payload, + occurred_at: t.finished_at ?? t.started_at, + }); + } + + if (t.dashboard_id) { + const payload: Record = {}; + put(payload, 'dashboard_id', t.dashboard_id); + put(payload, 'widget_count', t.widget_count); + put(payload, 'duration_ms', t.duration_ms); + put(payload, 'surface', t.surface ?? 'dashboard'); + events.push({ event_type: 'aud0.dashboard_viewed', payload, occurred_at: t.started_at }); + } + + return events; +} + +/** Every payload key this producer can ever emit. Pinned by test. */ +export const EMITTABLE_PAYLOAD_KEYS = [ + 'query_id', + 'query_hash', + 'dataset_id', + 'engine', + 'surface', + 'filter_keys', + 'duration_ms', + 'row_count', + 'bytes_scanned', + 'cache_hit', + 'error_code', + 'metric_name', + 'dimension_keys', + 'granularity', + 'dashboard_id', + 'widget_count', +] as const; diff --git a/tests/analytics/bi-slo.test.ts b/tests/analytics/bi-slo.test.ts new file mode 100644 index 0000000..1dc5df2 --- /dev/null +++ b/tests/analytics/bi-slo.test.ts @@ -0,0 +1,293 @@ +/** + * tests/analytics/bi-slo.test.ts + * =============================== + * BI-P103 — BI SLO, and the AUD0 usage events it is measured from. + * + * Two failure modes are attacked directly, because both look like a healthy + * dashboard: + * + * 1. AN SLO REPORT OVER SILENCE. Zero events must not read as 100% + * availability. Every objective in an empty window must come back + * `insufficient_data`, and `fully_measured` must be false. + * 2. A PRODUCER THAT LEAKS. The emitter is handed an execution record + * carrying SQL text, a database error message quoting a personnummer, + * filter VALUES, a tenant id and a result sample — everything AUD0's + * BI-P076 ingest exists to refuse — and none of it may appear in any + * emitted payload. The test greps the serialized events, and first + * proves the grep finds those strings in the INPUT so a clean result + * cannot be a broken detector. + */ +import { describe, expect, it } from 'vitest'; +import { + BI_SLO_CATALOGUE, + evaluateBiSlos, + percentile, + summariseSloReport, + type SloResult, +} from '@/src/analytics/slo/bi-slo'; +import { + biUsageEventsForExecution, + EMITTABLE_PAYLOAD_KEYS, + type Aud0BiEvent, + type ExecutionTelemetry, +} from '@/src/analytics/slo/usage-events'; + +const FROM = new Date('2026-09-07T00:00:00.000Z'); +const TO = new Date('2026-09-07T23:59:59.000Z'); + +function queryEvents(args: { + n: number; + surface: 'dashboard' | 'explore'; + outcome: 'completed' | 'failed' | 'running'; + durationMs: (i: number) => number; + cacheHit?: boolean; + idPrefix?: string; +}): Aud0BiEvent[] { + const out: Aud0BiEvent[] = []; + for (let i = 0; i < args.n; i += 1) { + const t: ExecutionTelemetry = { + query_id: `${args.idPrefix ?? args.surface}_${i}`, + surface: args.surface, + state: args.outcome === 'completed' ? 'succeeded' : args.outcome === 'failed' ? 'failed' : 'running', + started_at: new Date(FROM.getTime() + i * 1000).toISOString(), + finished_at: new Date(FROM.getTime() + i * 1000 + args.durationMs(i)).toISOString(), + duration_ms: args.durationMs(i), + ...(args.cacheHit !== undefined ? { cache_hit: args.cacheHit } : {}), + ...(args.outcome === 'failed' ? { error_code: 'TIMEOUT' } : {}), + }; + out.push(...biUsageEventsForExecution(t)); + } + return out; +} + +function byId(results: SloResult[], id: string): SloResult { + const r = results.find((x) => x.objective_id === id); + if (!r) throw new Error(`no result for ${id}`); + return r; +} + +describe('BI-P103 — an SLO over silence is not a met SLO', () => { + it('an empty window makes every measurable objective insufficient_data, never 100%', () => { + const report = evaluateBiSlos({ events: [], from: FROM, to: TO }); + expect(report.fully_measured).toBe(false); + for (const r of report.results) { + expect(['insufficient_data', 'not_measurable_here']).toContain(r.status); + expect(r.status).not.toBe('met'); + } + const summary = summariseSloReport(report); + expect(summary.met).toBe(0); + expect(summary.measured_fraction).toBe(0); + }); + + it('a started query that never finished is unresolved, not a success', () => { + const events = queryEvents({ n: 200, surface: 'dashboard', outcome: 'running', durationMs: () => 100 }); + const report = evaluateBiSlos({ events, from: FROM, to: TO }); + expect(report.unresolved_queries).toBe(200); + expect(byId(report.results, 'dash.availability').status).toBe('insufficient_data'); + expect(byId(report.results, 'dash.availability').observed).toBeUndefined(); + }); + + it('a percentile below min_samples is reported WITH its sample size and not judged', () => { + const events = queryEvents({ + n: 12, + surface: 'explore', + outcome: 'completed', + durationMs: (i) => 1000 + i * 10, + }); + const p99 = byId(evaluateBiSlos({ events, from: FROM, to: TO }).results, 'explore.p99'); + expect(p99.status).toBe('insufficient_data'); + expect(p99.low_confidence).toBe(true); + expect(p99.sample_size).toBe(12); + expect(p99.observed).toBeDefined(); // reported… + expect(p99.detail).toMatch(/not a percentile/); // …with the caveat attached + }); + + it('freshness and delivery are declared not_measurable_here rather than proxied', () => { + const report = evaluateBiSlos({ events: [], from: FROM, to: TO }); + const freshness = byId(report.results, 'dash.freshness'); + const delivery = byId(report.results, 'report.delivery_sla'); + expect(freshness.status).toBe('not_measurable_here'); + expect(freshness.detail).toMatch(/analytics_freshness_watermark/); + expect(delivery.status).toBe('not_measurable_here'); + expect(delivery.detail).toMatch(/n0tify_delivery_receipt/); + }); +}); + +describe('BI-P103 — real measurement from real events', () => { + it('computes availability, error rate and both dashboard latencies', () => { + const events = [ + ...queryEvents({ + n: 180, + surface: 'dashboard', + outcome: 'completed', + cacheHit: false, + durationMs: (i) => 500 + i * 10, + idPrefix: 'cold', + }), + ...queryEvents({ + n: 120, + surface: 'dashboard', + outcome: 'completed', + cacheHit: true, + durationMs: () => 120, + idPrefix: 'warm', + }), + ...queryEvents({ + n: 3, + surface: 'dashboard', + outcome: 'failed', + durationMs: () => 900, + idPrefix: 'bad', + }), + ]; + const { results } = evaluateBiSlos({ events, from: FROM, to: TO }); + + const availability = byId(results, 'dash.availability'); + expect(availability.sample_size).toBe(303); + expect(availability.observed).toBeCloseTo(300 / 303, 6); + expect(availability.status).toBe('breached'); // 0.9901 < 0.995 target + + const errorRate = byId(results, 'dash.error_rate'); + expect(errorRate.observed).toBeCloseTo(3 / 303, 6); + + const cold = byId(results, 'dash.p95_first_query'); + expect(cold.sample_size).toBe(183); // 180 cold + 3 failures, which had no cache hit + expect(cold.status).toBe('met'); + + const cached = byId(results, 'dash.p95_cached'); + expect(cached.sample_size).toBe(120); + expect(cached.observed).toBe(120); + expect(cached.status).toBe('met'); + }); + + it('a breach is reported as a breach', () => { + const events = queryEvents({ + n: 60, + surface: 'dashboard', + outcome: 'completed', + cacheHit: false, + durationMs: () => 9000, + }); + expect( + byId(evaluateBiSlos({ events, from: FROM, to: TO }).results, 'dash.p95_first_query'), + ).toMatchObject({ status: 'breached', observed: 9000 }); + }); + + it('events outside the window do not count', () => { + const events = queryEvents({ n: 200, surface: 'dashboard', outcome: 'completed', durationMs: () => 100 }); + const report = evaluateBiSlos({ + events, + from: new Date('2026-09-08T00:00:00.000Z'), + to: new Date('2026-09-08T23:59:59.000Z'), + }); + expect(byId(report.results, 'dash.availability').status).toBe('insufficient_data'); + }); + + it('percentile is nearest-rank — it never invents a value nobody experienced', () => { + const sample = [10, 20, 30, 40]; + expect(percentile(sample, 0.95)).toBe(40); + expect(percentile(sample, 0.5)).toBe(20); + expect(percentile([], 0.95)).toBeUndefined(); + }); + + it('the catalogue covers all three workload classes and every field BI-P103 names', () => { + const kinds = new Set(BI_SLO_CATALOGUE.map((o) => `${o.workload}:${o.kind}`)); + for (const expected of [ + 'interactive_dashboard:availability', + 'interactive_dashboard:latency_p95_first', + 'interactive_dashboard:latency_p95_cached', + 'interactive_dashboard:error_rate', + 'interactive_dashboard:freshness', + 'exploration:latency_p95', + 'exploration:latency_p99', + 'exploration:max_result_rows', + 'exploration:timeout', + 'report:completion_sla', + 'report:delivery_sla', + ]) { + expect(kinds.has(expected), expected).toBe(true); + } + }); +}); + +describe('BI-P103 — the producer cannot leak a result into AUD0', () => { + const hostile: ExecutionTelemetry = { + query_id: 'q_1', + compiled_query_hash: 'f'.repeat(64), + compiled_sql: "SELECT * FROM customers WHERE personnummer = '19800101-1234'", + error_message: 'duplicate key value violates unique constraint (personnummer)=(19800101-1234)', + error_code: 'CONFLICT', + tenant_id: 't_alpha', + dataset_id: 'ds_sales', + engine: 'clickhouse', + surface: 'dashboard', + state: 'failed', + started_at: '2026-09-07T10:00:00.000Z', + finished_at: '2026-09-07T10:00:02.000Z', + duration_ms: 2000, + filter_keys: ['region'], + filter_values: { region: ['Gotland'] }, + result_sample: [{ customer: 'Anna Andersson', revenue: 42 }], + metric_name: 'revenue', + dashboard_id: 'dash_sales', + widget_count: 4, + }; + + it('emits none of the forbidden material', () => { + const events = biUsageEventsForExecution(hostile); + const serialized = JSON.stringify(events); + + // The detector must be able to find these strings — they are in the input. + const input = JSON.stringify(hostile); + for (const secret of ['19800101-1234', 'Anna Andersson', 'Gotland', 't_alpha', 'SELECT *']) { + expect(input, `detector check for ${secret}`).toContain(secret); + } + for (const secret of [ + '19800101-1234', + 'Anna Andersson', + 'Gotland', + 't_alpha', + 'SELECT *', + 'duplicate key', + ]) { + expect(serialized, secret).not.toContain(secret); + } + }); + + it('every emitted key is on the allowlist AUD0 accepts', () => { + const events = biUsageEventsForExecution(hostile); + for (const e of events) { + for (const key of Object.keys(e.payload)) { + expect(EMITTABLE_PAYLOAD_KEYS, `${e.event_type}.${key}`).toContain(key); + } + } + }); + + it('a query_hash that is not a sha256 digest is dropped, not reshaped', () => { + const events = biUsageEventsForExecution({ ...hostile, compiled_query_hash: 'SELECT 1' }); + for (const e of events) expect(e.payload.query_hash).toBeUndefined(); + }); + + it('a running query emits only "started" — it is neither a success nor a failure', () => { + const events = biUsageEventsForExecution({ + ...hostile, + state: 'running', + metric_name: undefined, + dashboard_id: undefined, + }); + expect(events.map((e) => e.event_type)).toEqual(['aud0.analytics_query_started']); + }); + + it('a cancelled query is reported as failed with a CODE', () => { + const events = biUsageEventsForExecution({ + ...hostile, + state: 'cancelled', + error_code: undefined, + metric_name: undefined, + dashboard_id: undefined, + }); + const failed = events.find((e) => e.event_type === 'aud0.analytics_query_failed'); + expect(failed?.payload.error_code).toBe('CANCELLED'); + expect(failed?.payload.error_message).toBeUndefined(); + }); +});