diff --git a/evidence/bi-p103/baseline.json b/evidence/bi-p103/baseline.json new file mode 100644 index 0000000..019dadc --- /dev/null +++ b/evidence/bi-p103/baseline.json @@ -0,0 +1,173 @@ +{ + "prompt": "BI-P103", + "title": "First measured BI SLO baselines", + "recorded_at": "2026-09-07T06:26:43.457Z", + "host_note": "One idle host, local ClickHouse, warm page cache, single-threaded client. A floor for the shape of the distribution on this fixture — not a production SLA.", + "clickhouse_version": "26.3.10.60", + "fixture": { + "rows": 400000, + "table": "slo_fact", + "partitions": "MergeTree ORDER BY (tenant_id, bucket, ts)" + }, + "workload": { + "cold_dashboard_queries": 120, + "cached_dashboard_queries": 120, + "exploration_queries": 220, + "report_queries": 0 + }, + "window": { + "from": "2026-09-07T06:26:39.422Z", + "to": "2026-09-07T06:26:43.447Z" + }, + "catalogue_size": 11, + "fully_measured": false, + "unresolved_queries": 0, + "summary": { + "met": 7, + "breached": 0, + "insufficient_data": 1, + "not_measurable_here": 3, + "measured_fraction": 0.6363636363636364 + }, + "results": [ + { + "objective_id": "dash.availability", + "workload": "interactive_dashboard", + "kind": "availability", + "status": "met", + "target": 0.995, + "unit": "ratio", + "observed": 1, + "sample_size": 240, + "low_confidence": false, + "detail": "0 failed of 240 terminal queries" + }, + { + "objective_id": "dash.p95_first_query", + "workload": "interactive_dashboard", + "kind": "latency_p95_first", + "status": "met", + "target": 3000, + "unit": "ms", + "observed": 8.893520999999964, + "sample_size": 120, + "low_confidence": false, + "detail": "p95 over 120 samples" + }, + { + "objective_id": "dash.p95_cached", + "workload": "interactive_dashboard", + "kind": "latency_p95_cached", + "status": "met", + "target": 400, + "unit": "ms", + "observed": 0.024498999999877924, + "sample_size": 120, + "low_confidence": false, + "detail": "p95 over 120 samples" + }, + { + "objective_id": "dash.error_rate", + "workload": "interactive_dashboard", + "kind": "error_rate", + "status": "met", + "target": 0.005, + "unit": "ratio", + "observed": 0, + "sample_size": 240, + "low_confidence": false, + "detail": "0 failed of 240 terminal queries" + }, + { + "objective_id": "dash.freshness", + "workload": "interactive_dashboard", + "kind": "freshness", + "status": "not_measurable_here", + "target": 900, + "unit": "seconds", + "sample_size": 0, + "low_confidence": false, + "detail": "measured from analytics_freshness_watermark; BI usage events cannot answer it — Data age is a property of the DATASET (BI-P026 freshness, BI-P105 watermark), not of a usage event. Usage events cannot measure it and this evaluator does not pretend to." + }, + { + "objective_id": "explore.p95", + "workload": "exploration", + "kind": "latency_p95", + "status": "met", + "target": 10000, + "unit": "ms", + "observed": 17.072708000000148, + "sample_size": 220, + "low_confidence": false, + "detail": "p95 over 220 samples" + }, + { + "objective_id": "explore.p99", + "workload": "exploration", + "kind": "latency_p99", + "status": "met", + "target": 60000, + "unit": "ms", + "observed": 21.640736000000288, + "sample_size": 220, + "low_confidence": false, + "detail": "p99 over 220 samples" + }, + { + "objective_id": "explore.max_result", + "workload": "exploration", + "kind": "max_result_rows", + "status": "met", + "target": 1000000, + "unit": "rows", + "observed": 96, + "sample_size": 220, + "low_confidence": false, + "detail": "largest result seen in the window" + }, + { + "objective_id": "explore.timeout", + "workload": "exploration", + "kind": "timeout", + "status": "not_measurable_here", + "target": 300, + "unit": "seconds", + "sample_size": 0, + "low_confidence": false, + "detail": "measured from engine_configuration; BI usage events cannot answer it — The configured ceiling (BI-P012 settings profile), not an observation." + }, + { + "objective_id": "report.completion_sla", + "workload": "report", + "kind": "completion_sla", + "status": "insufficient_data", + "target": 900000, + "unit": "ms", + "sample_size": 0, + "low_confidence": false, + "detail": "no report_generated events" + }, + { + "objective_id": "report.delivery_sla", + "workload": "report", + "kind": "delivery_sla", + "status": "not_measurable_here", + "target": 300000, + "unit": "ms", + "sample_size": 0, + "low_confidence": false, + "detail": "measured from n0tify_delivery_receipt; BI usage events cannot answer it — Produced is not delivered. aud0 has no delivery event; the receipt is N0TIFY's, so this objective is declared and reported unmeasured rather than proxied by completion." + } + ], + "measured_but_narrower_than_production": [ + "dash.p95_cached is the IN-PROCESS reference cache (InMemoryResultCache). A Valkey deployment adds a network round trip, so 0.03 ms here is a floor and not the number a user will see.", + "Every latency is the ENGINE round trip only: no HTTP layer, no CL0UD authorization, no serialization to the caller. The end-to-end p95 is larger by whatever those cost.", + "One tenant, one client, no concurrency. Contention is not represented." + ], + "not_measured_and_why": [ + "dash.freshness — a property of the DATASET (BI-P026 watermark), not of a usage event", + "explore.timeout — the configured BI-P012 ceiling, not an observation", + "report.completion_sla — nothing in 0pt emits aud0.report_generated yet, so the sample is 0", + "report.delivery_sla — delivery is N0TIFY's receipt; aud0 has no delivery event" + ] +} diff --git a/src/analytics/slo/bi-slo.ts b/src/analytics/slo/bi-slo.ts index b4bad4b..c172c94 100644 --- a/src/analytics/slo/bi-slo.ts +++ b/src/analytics/slo/bi-slo.ts @@ -78,8 +78,18 @@ export interface SloObjective { * The catalogue, field-for-field with what BI-P103 lists per workload class. * Targets are STARTING POINTS chosen to be honest about what has been * measured: they are the thresholds the evaluator compares against, and - * evidence/bi-p103 records the first baselines rather than pretending the - * targets were derived from a year of production traffic. + * evidence/bi-p103/baseline.json records the first baselines rather than + * pretending the targets were derived from a year of production traffic. + * + * Those baselines are produced by + * tests/integration/bi-p103-slo-baseline.integration.test.ts, which runs 460 + * real queries against a live ClickHouse and feeds them through this + * evaluator. Read the file's own caveats before quoting a number from it: the + * cached path measured there is the in-process reference cache rather than + * Valkey, every latency is the engine round trip with no HTTP or + * authorization layer around it, and there is no concurrency. Seven of the + * eleven objectives produced a number; the other four say why they could not, + * and `fully_measured` is false. */ export const BI_SLO_CATALOGUE: readonly SloObjective[] = [ // ── interactive dashboard ──────────────────────────────────────────────── diff --git a/tests/integration/bi-p103-slo-baseline.integration.test.ts b/tests/integration/bi-p103-slo-baseline.integration.test.ts new file mode 100644 index 0000000..6fecacd --- /dev/null +++ b/tests/integration/bi-p103-slo-baseline.integration.test.ts @@ -0,0 +1,341 @@ +/** + * tests/integration/bi-p103-slo-baseline.integration.test.ts + * =========================================================== + * BI-P103 — "Measured baselines exist." + * + * `src/analytics/slo/bi-slo.ts` said the first baselines were recorded in + * `evidence/bi-p103`. There was no `evidence/bi-p103`. The catalogue and the + * evaluator were both real and both well tested — against SYNTHETIC events. + * A synthetic event carries whatever duration the test author typed, so a + * suite built only on those proves the arithmetic and nothing about the + * system: it cannot tell you a dashboard query takes 8 ms or 8 s. + * + * So this harness produces the baselines from queries that actually ran: + * + * * ~120 COLD dashboard queries, each a different filter so the BI-P020 + * cache genuinely misses, executed against a live ClickHouse; + * * the same ~120 again through the SAME shipped cache, now hitting, so + * `dash.p95_cached` is a measured cache path and not an assumption; + * * 220 exploration queries, enough for `explore.p99`'s min_samples of 200 + * to be met rather than reported low-confidence. + * + * Every duration is wall-clock around the real call. The telemetry goes + * through the SHIPPED producer (`biUsageEventsForExecution`) and the SHIPPED + * evaluator (`evaluateBiSlos`) — not a copy — so the evidence is produced by + * the code under test. + * + * WHAT THE BASELINE DELIBERATELY DOES NOT CONTAIN. `dash.freshness`, + * `explore.timeout` and `report.delivery_sla` are not measurable from usage + * events and the evaluator says so; `report.completion_sla` has no producer in + * 0pt at all (nothing here emits `aud0.report_generated`). They appear in the + * evidence with their real status rather than being quietly dropped, because a + * baseline table that lists only the rows that worked is how an estate comes + * to believe it measures more than it does. + * + * These numbers describe THIS fixture on THIS machine against a local + * ClickHouse with warm page cache. They are a floor for the shape of the + * distribution, not a production SLA — and the evidence file says which parts + * are narrower than production: the cached path is the in-process reference + * cache rather than Valkey, every latency is the engine round trip with no + * HTTP or authorization layer around it, and there is no concurrency. + * + * CLICKHOUSE_URL=http://127.0.0.1:28123 CLICKHOUSE_AUTH=default:test \ + * pnpm vitest run tests/integration/bi-p103-slo-baseline.integration.test.ts + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { ClickHouseAnalyticsEngine } from '@/src/analytics/engine/clickhouse-engine'; +import { InMemoryResultCache, type CacheKeyComponents } from '@/src/analytics/cache/query-cache'; +import { compiledQueryHash } from '@/src/analytics/execution/query-execution-record'; +import { + biUsageEventsForExecution, + type Aud0BiEvent, + type ExecutionTelemetry, +} from '@/src/analytics/slo/usage-events'; +import { BI_SLO_CATALOGUE, evaluateBiSlos, summariseSloReport } from '@/src/analytics/slo/bi-slo'; + +const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; +const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; +const run = CLICKHOUSE_URL ? describe : describe.skip; + +const DB = `opt_p103_${Date.now().toString(36)}`; +const TENANT = 't_alpha'; +const ROWS = 400_000; +const COLD_DASHBOARD_QUERIES = 120; +const EXPLORE_QUERIES = 220; + +const headers: Record = CLICKHOUSE_AUTH + ? { Authorization: `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}` } + : {}; + +async function ch(sql: string, body?: string): Promise { + const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); + url.searchParams.set('query', sql); + const res = await fetch(url.toString(), { method: 'POST', headers, body: body ?? '' }); + const text = await res.text(); + if (!res.ok) throw new Error(`ClickHouse ${res.status}: ${text.slice(0, 300)}`); + return text; +} + +interface Measured { + rows: number; + ms: number; +} + +/** One real round trip, wall-clock measured. */ +async function timed(sql: string): Promise { + const started = performance.now(); + const text = await ch(`${sql} FORMAT JSON`); + const ms = performance.now() - started; + const parsed = JSON.parse(text) as { data: unknown[] }; + return { rows: parsed.data.length, ms }; +} + +const events: Aud0BiEvent[] = []; + +function record(t: ExecutionTelemetry): void { + events.push(...biUsageEventsForExecution(t)); +} + +function keyFor(i: number, cachedShape: string): CacheKeyComponents { + return { + tenant_id: TENANT, + dataset_id: 'ds_slo', + semantic_model_version: 1, + metric_versions: { m_amount: 1 }, + compiled_query_hash: compiledQueryHash(cachedShape, { i: String(i) }), + data_snapshot: 'v1', + format: 'json', + }; +} + +let windowFrom: Date; +let windowTo: Date; + +run('BI-P103 — measured baselines, from queries that actually ran', () => { + beforeAll(async () => { + await new ClickHouseAnalyticsEngine({ + baseUrl: CLICKHOUSE_URL!, + basicAuth: CLICKHOUSE_AUTH, + database: DB, + dimensionCatalogs: { [TENANT]: { region: 'region' } }, + }).ensureSchema(); + + // A fact table large enough that a scan is not free. Generated inside + // ClickHouse so the fixture cost is not measured as query latency. + await ch( + `CREATE TABLE ${DB}.slo_fact (tenant_id LowCardinality(String), region LowCardinality(String), ` + + `bucket UInt16, ts DateTime, amount Float64) ENGINE = MergeTree ORDER BY (tenant_id, bucket, ts)`, + ); + await ch( + `INSERT INTO ${DB}.slo_fact SELECT 't_alpha', ['SE','NO','DK','FI'][(number % 4) + 1], ` + + `toUInt16(number % 500), toDateTime('2026-09-01 00:00:00') + (number % 86400), ` + + `toFloat64(number % 997) FROM numbers(${ROWS})`, + ); + await ch(`OPTIMIZE TABLE ${DB}.slo_fact FINAL`); + }, 300_000); + + afterAll(async () => { + if (CLICKHOUSE_URL) await ch(`DROP DATABASE IF EXISTS ${DB}`); + }, 60_000); + + it('measures a cold and a cached dashboard path, and an exploration path', async () => { + windowFrom = new Date(); + const cache = new InMemoryResultCache(); + const shape = `SELECT region, sum(amount) FROM ${DB}.slo_fact WHERE tenant_id = ? AND bucket = ? GROUP BY region`; + + // ── cold: every query a different bucket, so the cache genuinely misses ── + for (let i = 0; i < COLD_DASHBOARD_QUERIES; i++) { + const components = keyFor(i, shape); + const started = new Date().toISOString(); + const before = await cache.get(components); + expect(before).toBeUndefined(); // a "cold" sample served from cache is not cold + const m = await timed( + `SELECT region, sum(amount) AS total FROM ${DB}.slo_fact ` + + `WHERE tenant_id = 't_alpha' AND bucket = ${i} GROUP BY region ORDER BY region`, + ); + await cache.set(components, { rows: m.rows }, { ttlSeconds: 300, rowCount: m.rows }); + record({ + query_id: `q_cold_${i}`, + surface: 'dashboard', + dataset_id: 'ds_slo', + engine: 'clickhouse', + state: 'succeeded', + started_at: started, + finished_at: new Date().toISOString(), + duration_ms: m.ms, + row_count: m.rows, + cache_hit: false, + }); + } + + // ── cached: the same questions, through the same shipped cache ────────── + for (let i = 0; i < COLD_DASHBOARD_QUERIES; i++) { + const started = new Date().toISOString(); + const t0 = performance.now(); + const hit = await cache.get(keyFor(i, shape)); + const ms = performance.now() - t0; + expect(hit).toBeDefined(); // a miss recorded as a hit would flatter the baseline + record({ + query_id: `q_hot_${i}`, + surface: 'dashboard', + dataset_id: 'ds_slo', + engine: 'clickhouse', + state: 'succeeded', + started_at: started, + finished_at: new Date().toISOString(), + duration_ms: ms, + row_count: hit?.row_count ?? 0, + cache_hit: true, + }); + } + + // ── exploration: wider, ad-hoc shapes, enough samples for a real p99 ──── + for (let i = 0; i < EXPLORE_QUERIES; i++) { + const started = new Date().toISOString(); + const m = await timed( + `SELECT region, toStartOfHour(ts) AS h, sum(amount) AS total, count() AS n ` + + `FROM ${DB}.slo_fact WHERE tenant_id = 't_alpha' AND bucket >= ${i % 400} ` + + `GROUP BY region, h ORDER BY region, h LIMIT 5000`, + ); + record({ + query_id: `q_explore_${i}`, + surface: 'explore', + dataset_id: 'ds_slo', + engine: 'clickhouse', + state: 'succeeded', + started_at: started, + finished_at: new Date().toISOString(), + duration_ms: m.ms, + row_count: m.rows, + cache_hit: false, + }); + } + windowTo = new Date(); + + expect(events.length).toBeGreaterThan(2 * (COLD_DASHBOARD_QUERIES * 2 + EXPLORE_QUERIES) - 1); + }, 600_000); + + it('a REAL engine error becomes a failed event and moves the error rate', async () => { + // Proves the availability and error-rate paths are not measuring silence. + // Kept OUT of the baseline window so the recorded baseline describes the + // system's normal behaviour rather than a deliberately broken query. + const started = new Date().toISOString(); + let code: string | undefined; + const t0 = performance.now(); + try { + await ch(`SELECT no_such_column FROM ${DB}.slo_fact LIMIT 1 FORMAT JSON`); + } catch { + code = 'ENGINE_ERROR'; + } + expect(code).toBe('ENGINE_ERROR'); + + const failed = biUsageEventsForExecution({ + query_id: 'q_fail_1', + surface: 'dashboard', + dataset_id: 'ds_slo', + engine: 'clickhouse', + state: 'failed', + started_at: started, + finished_at: new Date().toISOString(), + duration_ms: performance.now() - t0, + error_code: code, + error_message: 'Unknown expression identifier no_such_column', + }); + expect(failed.map((e) => e.event_type)).toContain('aud0.analytics_query_failed'); + // The engine's message never becomes telemetry. + expect(JSON.stringify(failed)).not.toContain('no_such_column'); + + const withFailure = evaluateBiSlos({ + events: [...events, ...failed], + from: windowFrom, + to: new Date(), + }); + const availability = withFailure.results.find((r) => r.objective_id === 'dash.availability'); + expect(availability?.sample_size).toBeGreaterThanOrEqual(100); + expect(availability?.observed).toBeLessThan(1); + }); + + it('writes the baseline, including every objective it could NOT measure', async () => { + const report = evaluateBiSlos({ events, from: windowFrom, to: windowTo }); + const summary = summariseSloReport(report); + + // The three latency objectives must be real numbers over real samples — + // an evidence file full of insufficient_data is not a baseline. + for (const id of ['dash.p95_first_query', 'dash.p95_cached', 'explore.p95', 'explore.p99']) { + const r = report.results.find((x) => x.objective_id === id); + expect(r?.observed, `${id} produced no number`).toBeTypeOf('number'); + expect(r?.low_confidence, `${id} was computed from too few samples`).not.toBe(true); + } + // ...and the cached path must actually be faster than the cold one, or the + // two were not measuring different things. + const cold = report.results.find((r) => r.objective_id === 'dash.p95_first_query')?.observed ?? 0; + const cached = report.results.find((r) => r.objective_id === 'dash.p95_cached')?.observed ?? 0; + expect(cached).toBeLessThan(cold); + + const dir = path.join(process.cwd(), 'evidence', 'bi-p103'); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, 'baseline.json'), + `${JSON.stringify( + { + prompt: 'BI-P103', + title: 'First measured BI SLO baselines', + recorded_at: new Date().toISOString(), + host_note: + 'One idle host, local ClickHouse, warm page cache, single-threaded client. A floor for the ' + + 'shape of the distribution on this fixture — not a production SLA.', + clickhouse_version: JSON.parse(await ch('SELECT version() AS v FORMAT JSON')).data[0].v, + fixture: { + rows: ROWS, + table: 'slo_fact', + partitions: 'MergeTree ORDER BY (tenant_id, bucket, ts)', + }, + workload: { + cold_dashboard_queries: COLD_DASHBOARD_QUERIES, + cached_dashboard_queries: COLD_DASHBOARD_QUERIES, + exploration_queries: EXPLORE_QUERIES, + report_queries: 0, + }, + window: { from: windowFrom.toISOString(), to: windowTo.toISOString() }, + catalogue_size: BI_SLO_CATALOGUE.length, + fully_measured: report.fully_measured, + unresolved_queries: report.unresolved_queries, + summary, + results: report.results.map((r) => ({ + objective_id: r.objective_id, + workload: r.workload, + kind: r.kind, + status: r.status, + target: r.target, + unit: r.unit, + observed: r.observed, + sample_size: r.sample_size, + low_confidence: r.low_confidence ?? false, + detail: r.detail, + })), + measured_but_narrower_than_production: [ + 'dash.p95_cached is the IN-PROCESS reference cache (InMemoryResultCache). A Valkey ' + + 'deployment adds a network round trip, so 0.03 ms here is a floor and not the number a ' + + 'user will see.', + 'Every latency is the ENGINE round trip only: no HTTP layer, no CL0UD authorization, no ' + + 'serialization to the caller. The end-to-end p95 is larger by whatever those cost.', + 'One tenant, one client, no concurrency. Contention is not represented.', + ], + not_measured_and_why: [ + 'dash.freshness — a property of the DATASET (BI-P026 watermark), not of a usage event', + 'explore.timeout — the configured BI-P012 ceiling, not an observation', + 'report.completion_sla — nothing in 0pt emits aud0.report_generated yet, so the sample is 0', + "report.delivery_sla — delivery is N0TIFY's receipt; aud0 has no delivery event", + ], + }, + null, + 2, + )}\n`, + ); + + // The file must be honest about the unmeasured half. + expect(report.fully_measured).toBe(false); + }); +});