/** * tests/integration/semantic-parity.integration.test.ts * ====================================================== * BI-P029 — the parity test `src/analytics/metrics/semantic-provider.ts` has * been citing. * * IT DID NOT EXIST. The provider's header named this path as the proof of the * one DoD surface it claimed was closed, and `providerReadiness().proven` * asserted that "0pt's metric compiler and an independent spec-driven renderer * return the same number for the golden metric against live ClickHouse data". * Nothing in the repository executed that comparison. A citation to a file * that is not there is the same failure as a green check that gates nothing: * it survives review because the sentence reads like evidence. * * So this file is the comparison, run for real: * * PATH A parseAnalyticsQueryRequest -> resolveAnalyticsQuery -> * compileRequestToClickHouse -> live ClickHouse. * PATH B the same ResolvedQuery -> compileRequestToSemantic -> * MetricFlowProvider.compile -> `renderIndependently` below, which * reads ONLY the provider's structured query object and the registry * model, writes its own SQL with its own quoting, and shares no line * of code with compile-clickhouse.ts -> the same live ClickHouse. * * The two paths must return the same numbers. `renderIndependently` inlines * literals where compile-clickhouse binds parameters, groups by the physical * column expression rather than by a compiler alias, and orders in SQL rather * than by alias — so the SQL strings genuinely differ, and the test asserts * that they do. Two identical strings would make the parity claim vacuous. * * WHAT THIS STILL DOES NOT PROVE, and `providerReadiness()` still says so: * MetricFlow itself is not executed here (no Python runtime is provisioned for * this repo), and neither the Superset path nor the AI Analyst path is * compared. Three of BI-P029's four DoD surfaces remain open. This closes the * first one honestly rather than closing all four by assertion. * * CLICKHOUSE_URL=http://127.0.0.1:28123 CLICKHOUSE_AUTH=default:test \ * pnpm vitest run tests/integration/semantic-parity.integration.test.ts * * It CREATEs and DROPs its own database. Never point it at the b0ts * ClickHouse or at any quarantined dataset. */ import fs from 'node:fs'; import path from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { ClickHouseAnalyticsEngine } from '@/src/analytics/engine/clickhouse-engine'; import { parseAnalyticsQueryRequest, queryResolutionFrom, resolveAnalyticsQuery, type ResolvedQuery, } from '@/src/analytics/contract/query-request'; import { compileRequestToClickHouse } from '@/src/analytics/contract/compile-clickhouse'; import { compileRequestToSemantic } from '@/src/analytics/contract/compile-semantic'; import { MetricFlowProvider } from '@/src/analytics/metrics/semantic-provider'; import { InMemorySemanticModelStore, SemanticModelRegistry, type SemanticModelVersion, } from '@/src/analytics/metrics/semantic-registry'; import type { MetricSpec } from '@/src/analytics/metrics/metric-model'; /** * Raw events carry a 30-day TTL (src/analytics/engine/schema.ts). Fixed * historical timestamps silently expire and make live-ClickHouse suites read * 0 rows, so seed rows are stamped relative to now (UTC, ClickHouse * DateTime64(3) text form). */ const SEED_EVENT_TIME = new Date(Date.now() - 2 * 3600_000).toISOString().replace("T", " ").replace("Z", ""); const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; // Collection-time, never inside a hook: `describe.skipIf(flagSetInBeforeAll)` // evaluates its flag before any hook has run and therefore skips ALWAYS. const run = CLICKHOUSE_URL ? describe : describe.skip; const DB = `opt_parity_${Date.now().toString(36)}`; const TENANT = 't_alpha'; const PROJECT = 'p_main'; /** * The golden metric. It carries a DEFINITION FILTER (`channel != 'internal'`) * on purpose: a metric whose meaning is only "sum this column" cannot tell a * parity test anything, because every compiler agrees about a bare sum. The * business meaning of net revenue is that internal transfers are not revenue, * and that meaning has to survive both compile paths or the two paths are * answering different questions with the same metric id. */ const REVENUE: MetricSpec = { id: 'm_revenue', name: 'net_revenue', description: 'Invoiced revenue net of credit notes, in SEK, recognised on invoice date.', type: 'simple', measure: 'amount_sek', aggregation: 'sum', dimensions: ['country', 'channel'], filters: [{ dimension: 'channel', operator: 'ne', values: ['internal'] }], timeDimension: 'invoiced_at', unit: 'SEK', currency: 'SEK', format: 'currency', owner: 'finance@siax.io', version: 3, status: 'ACTIVE', }; /** dimension name -> the key stored in `dim_names`. One mapping, both paths. */ const FIELD_OF: Record = { country: 'country_code', channel: 'sales_channel' }; function semanticModel(version = 1): Omit { return { model_id: 'sm_sales', version, tenant_id: TENANT, datasets: ['ds_sales'], relationships: [], metrics: [REVENUE], dimensions: [ { id: 'd_country', name: 'country', dataset_id: 'ds_sales', kind: 'categorical' }, { id: 'd_channel', name: 'channel', dataset_id: 'ds_sales', kind: 'categorical' }, { id: 'd_invoiced', name: 'invoiced_at', dataset_id: 'ds_sales', kind: 'time' }, ], valid_from: '2026-09-01T00:00:00Z', status: 'draft', }; } /** * The fixture. Deliberately arranged so that dropping the metric's definition * filter changes the answer for EVERY country — a fixture where internal rows * happen to sum to zero would let a broken compiler pass. */ const ROWS: Array<{ country: string; channel: string; amount: number }> = [ { country: 'SE', channel: 'direct', amount: 1000 }, { country: 'SE', channel: 'partner', amount: 250 }, { country: 'SE', channel: 'internal', amount: 9999 }, { country: 'NO', channel: 'direct', amount: 400 }, { country: 'NO', channel: 'internal', amount: 7777 }, { country: 'DK', channel: 'partner', amount: 125.5 }, { country: 'DK', channel: 'internal', amount: 3333 }, ]; /** What the metric MEANS, computed in TypeScript, independent of any SQL. */ const EXPECTED: Record = ROWS.filter((r) => r.channel !== 'internal').reduce( (acc, r) => ({ ...acc, [r.country]: (acc[r.country] ?? 0) + r.amount }), {} as Record, ); interface ChRow { [column: string]: string | number; } async function chQuery(sql: string, params: Record = {}): Promise { const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); const headers: Record = {}; if (CLICKHOUSE_AUTH) headers.Authorization = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}`; const res = await fetch(url.toString(), { method: 'POST', headers, body: sql }); const text = await res.text(); if (!res.ok) throw new Error(`ClickHouse ${res.status}: ${text.slice(0, 400)}`); if (!text.trim()) return []; return (JSON.parse(text) as { data: ChRow[] }).data; } async function chExec(sql: string, body?: string): Promise { const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); url.searchParams.set('query', sql); const headers: Record = {}; if (CLICKHOUSE_AUTH) headers.Authorization = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}`; const res = await fetch(url.toString(), { method: 'POST', headers, body: body ?? '' }); if (!res.ok) throw new Error(`ClickHouse ${res.status}: ${(await res.text()).slice(0, 400)}`); } // ── PATH B: the independent renderer ─────────────────────────────────────── /** Single-quote escaping, written here because this renderer shares nothing. */ function lit(value: string): string { return `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`; } function predicate(field: string, operator: string, values: string[]): string { const lhs = `dim_values[indexOf(dim_names, ${lit(field)})]`; switch (operator) { case 'eq': return `${lhs} = ${lit(values[0])}`; case 'ne': return `${lhs} <> ${lit(values[0])}`; case 'in': return `${lhs} IN (${values.map(lit).join(',')})`; case 'not_in': return `${lhs} NOT IN (${values.map(lit).join(',')})`; default: throw new Error(`independent renderer does not implement operator ${operator}`); } } /** * Render the provider's structured query into SQL, reading ONLY that object * and the registry model. It never sees the ResolvedQuery, the ClickHouse * compiler's output, or its parameter map. * * The aggregate is deliberately expressed differently from compile-clickhouse: * a `sum(if(...))` rather than a `sumIf(...)`, grouped by the raw expression * rather than by an alias. Same arithmetic, different SQL — which is the only * way the comparison means anything. */ function renderIndependently( compiled: ReturnType, model: SemanticModelVersion, ): string { const query = compiled.query as { metrics: string[]; group_by: string[]; where: Array<{ dimension: string; operator: string; values: string[] }>; limit: number; metric_filters?: Record>; }; const byId = new Map(model.metrics.map((m) => [m.id, m])); const selects: string[] = []; const groups: string[] = []; for (const dimension of query.group_by) { const expr = `dim_values[indexOf(dim_names, ${lit(FIELD_OF[dimension])})]`; selects.push(`${expr} AS ${dimension}`); groups.push(expr); } for (const metricId of query.metrics) { const spec = byId.get(metricId); if (!spec) throw new Error(`metric ${metricId} is not in the model version`); // The metric's own definition filters. The provider query carries them // already resolved onto physical keys; the registry model states them in // dimension names. Either way they are the metric's MEANING, and a // renderer that ignored them would compute a different number. const own = query.metric_filters?.[metricId] ? query.metric_filters[metricId].map((f) => ({ field: f.dimension, operator: f.operator, values: f.values, })) : spec.filters.map((f) => ({ field: FIELD_OF[f.dimension], operator: f.operator, values: f.values ?? [], })); const conditions = [ `metric = ${lit(spec.measure)}`, ...own.map((f) => predicate(f.field, f.operator, f.values)), ]; selects.push(`sum(if(${conditions.join(' AND ')}, value, 0)) AS ${spec.id}`); } const where = [ `tenant_id = ${lit(TENANT)}`, `project_id = ${lit(PROJECT)}`, `dataset_id = ${lit('ds_sales')}`, ...query.where.map((w) => predicate(FIELD_OF[w.dimension], w.operator, w.values ?? [])), ]; return ( `SELECT ${selects.join(', ')} FROM ${DB}.opt_analytics_event ` + `WHERE ${where.join(' AND ')} ` + (groups.length ? `GROUP BY ${groups.join(', ')} ORDER BY ${groups.join(', ')} ` : '') + `LIMIT ${query.limit} FORMAT JSON` ); } // ── the suite ────────────────────────────────────────────────────────────── let model: SemanticModelVersion; let resolved: ResolvedQuery; /** Filled by the regression test below; recorded in the evidence file. */ let preFix: Record = {}; run('BI-P029 — one golden metric, two independent compilers, one live ClickHouse', () => { beforeAll(async () => { const engine = new ClickHouseAnalyticsEngine({ baseUrl: CLICKHOUSE_URL!, basicAuth: CLICKHOUSE_AUTH, database: DB, dimensionCatalogs: { [TENANT]: FIELD_OF }, }); await engine.ensureSchema(); const body = ROWS.map((r, i) => JSON.stringify({ tenant_id: TENANT, project_id: PROJECT, dataset_id: 'ds_sales', metric: 'amount_sek', event_id: `e${i}`, event_time: SEED_EVENT_TIME, value: r.amount, dim_names: ['country_code', 'sales_channel'], dim_values: [r.country, r.channel], }), ).join('\n'); await chExec(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, body); model = await new SemanticModelRegistry(new InMemorySemanticModelStore()).publish(semanticModel(1)); resolved = resolveAnalyticsQuery( parseAnalyticsQueryRequest({ dataset: 'ds_sales', metrics: ['net_revenue'], dimensions: ['country'], orderBy: [{ ref: 'country', direction: 'asc' }], }), queryResolutionFrom({ dataset_id: 'ds_sales', semantic_model_version: 1, metrics: [ { name: 'net_revenue', metric_id: 'm_revenue', event_metric_key: 'amount_sek', aggregation: 'sum', allowed_dimensions: ['country', 'channel'], // The metric's OWN filters, resolved onto physical field names. definition_filters: REVENUE.filters.map((f) => ({ field_name: FIELD_OF[f.dimension], operator: f.operator, values: f.values ?? [], })), }, ], dimensions: [ { name: 'country', dimension_id: 'd_country', field_name: 'country_code' }, { name: 'channel', dimension_id: 'd_channel', field_name: 'sales_channel' }, ], }), ); }, 120_000); afterAll(async () => { if (CLICKHOUSE_URL) await chExec(`DROP DATABASE IF EXISTS ${DB}`); }, 60_000); it('the fixture is arranged so a dropped definition filter CHANGES every answer', () => { const withInternal = ROWS.reduce( (acc, r) => ({ ...acc, [r.country]: (acc[r.country] ?? 0) + r.amount }), {} as Record, ); for (const country of Object.keys(EXPECTED)) { expect(withInternal[country]).not.toBe(EXPECTED[country]); } }); it('PATH A — 0pt’s ClickHouse compiler returns the metric’s meaning', async () => { const compiled = compileRequestToClickHouse(resolved, { tenant_id: TENANT, project_id: PROJECT, database: DB, maxResultRows: 1000, }); const rows = await chQuery(compiled.sql, compiled.params); const got = Object.fromEntries(rows.map((r) => [String(r.dim_0), Number(r.metric_0)])); expect(got).toEqual(EXPECTED); }); it('PATH B — an independent renderer driven only by the provider spec agrees', async () => { const spec = compileRequestToSemantic(resolved); const compiled = new MetricFlowProvider().compile(spec, model); expect(compiled.canonical_metric_ids).toEqual(['m_revenue']); const rows = await chQuery(renderIndependently(compiled, model)); const got = Object.fromEntries(rows.map((r) => [String(r.country), Number(r.m_revenue)])); expect(got).toEqual(EXPECTED); }); /** * The regression this suite was written for. Until BI-P029's gap review, * `compileRequestToClickHouse` had no notion of a metric's own filters: * `ResolvedMetric` did not carry them and the compiler never asked. The * metric definition said "excluding internal transfers" and the executed * SQL did not, so `net_revenue` returned a number that looked plausible and * was ~9x too large for this fixture. * * Re-run here by stripping the definition filters from the resolution, so * the number the bug produced is measured by the committed test rather than * remembered from a console. */ it('a resolution WITHOUT the definition filters produces the pre-fix (wrong) number', async () => { const stripped: ResolvedQuery = { ...resolved, metrics: resolved.metrics.map((m) => ({ ...m, definition_filters: undefined })), }; const compiled = compileRequestToClickHouse(stripped, { tenant_id: TENANT, project_id: PROJECT, database: DB, maxResultRows: 1000, }); const rows = await chQuery(compiled.sql, compiled.params); preFix = Object.fromEntries(rows.map((r) => [String(r.dim_0), Number(r.metric_0)])); // Not merely different — OVERSTATED, on every country, by the internal // transfers the definition excludes. for (const country of Object.keys(EXPECTED)) { expect(preFix[country]).toBeGreaterThan(EXPECTED[country]); } expect(compiled.sql).not.toContain('mf0_0'); }); it('the golden metric returns the SAME number on both paths, within tolerance', async () => { const a = await chQuery( ...(() => { const c = compileRequestToClickHouse(resolved, { tenant_id: TENANT, project_id: PROJECT, database: DB, maxResultRows: 1000, }); return [c.sql, c.params] as const; })(), ); const b = await chQuery( renderIndependently(new MetricFlowProvider().compile(compileRequestToSemantic(resolved), model), model), ); const left = Object.fromEntries(a.map((r) => [String(r.dim_0), Number(r.metric_0)])); const right = Object.fromEntries(b.map((r) => [String(r.country), Number(r.m_revenue)])); expect(Object.keys(left).sort()).toEqual(Object.keys(right).sort()); for (const key of Object.keys(left)) { // Relative tolerance, because two different aggregate expressions over // Float64 may associate the additions differently. const denominator = Math.max(Math.abs(left[key]), 1); expect(Math.abs(left[key] - right[key]) / denominator).toBeLessThan(1e-9); } // Parity between two identical strings proves nothing. const sqlA = compileRequestToClickHouse(resolved, { tenant_id: TENANT, project_id: PROJECT, database: DB, maxResultRows: 1000, }).sql; const sqlB = renderIndependently( new MetricFlowProvider().compile(compileRequestToSemantic(resolved), model), model, ); expect(sqlA).not.toBe(sqlB); expect(sqlA).toContain('sumIf('); expect(sqlB).toContain('sum(if('); fs.mkdirSync(path.join(process.cwd(), 'evidence', 'bi-p029'), { recursive: true }); fs.writeFileSync( path.join(process.cwd(), 'evidence', 'bi-p029', 'semantic-parity.json'), `${JSON.stringify( { prompt: 'BI-P029', title: 'Golden metric parity: 0pt metric compiler vs an independent spec-driven renderer', recorded_at: new Date().toISOString(), clickhouse_version: (await chQuery('SELECT version() AS v FORMAT JSON'))[0]?.v, golden_metric: { id: REVENUE.id, name: REVENUE.name, definition_filters: REVENUE.filters, semantic_model: { model_id: model.model_id, version: model.version, hash: model.hash }, }, expected_from_typescript: EXPECTED, regression_guarded: { description: 'the same request compiled WITHOUT the metric definition filters — what 0pt returned before this change', result: preFix, }, path_a: { compiler: '0pt compileRequestToClickHouse', aggregate: 'sumIf', result: left }, path_b: { compiler: 'MetricFlowProvider.compile + independent renderer', aggregate: 'sum(if(...))', result: right, }, tolerance: '1e-9 relative', verdict: 'PASS', still_open: [ 'MetricFlow itself was not executed (no Python runtime provisioned for this repo)', 'the Superset integration path is not implemented', 'the AI Analyst path (inf0#29) has not been compared against this provider', ], }, null, 2, )}\n`, ); }); });