/** * tests/analytics/metric-governance.test.ts * ========================================== * BI-P028 (metric domain model), BI-P030 (semantic model registry), * BI-P031 (metric governance), BI-P029 (semantic provider — the parts not * needing a Python runtime). * * Each prompt's DoD is turned into something that can FAIL: * * BI-P028 "No production metric exists only inside a dashboard tool." * A dashboard panel carrying an inline aggregation / expression / * SQL is REFUSED, and so is one naming an unregistered metric. * * BI-P030 "MetricFlow can be replaced without breaking canonical SIAX * metric IDs." Two providers with different naming rules compile * the same registry version; the canonical id set must be * identical and the provider BINDINGS must differ — otherwise the * test would pass for a registry that simply has one provider. * * BI-P031 "Certified metric cannot be modified in-place; new version * required." Every semantic field is mutated one at a time and * each must be refused; every editorial field is mutated and each * must be allowed. */ import { describe, expect, it } from 'vitest'; import { SiaxError } from '@/src/platform/errors'; import { assertMetricEditAllowed, assertPanelUsesRegisteredMetric, assertValidMetricSpec, certifyMetric, metricMeaningHash, nextMetricVersion, METRIC_STATUSES, type GovernedMetric, type MetricSpec, } from '@/src/analytics/metrics/metric-model'; import { assertEngineNeutral, assertOnlySemanticModelFields, InMemorySemanticModelStore, SemanticModelRegistry, semanticModelHash, type SemanticModelVersion, } from '@/src/analytics/metrics/semantic-registry'; import { MetricFlowProvider, type SiaxSemanticProvider } from '@/src/analytics/metrics/semantic-provider'; import { compileRequestToSemantic } from '@/src/analytics/contract/compile-semantic'; import { parseAnalyticsQueryRequest, queryResolutionFrom, resolveAnalyticsQuery, } from '@/src/analytics/contract/query-request'; const REVENUE: MetricSpec = { id: 'm_revenue', name: 'net_revenue', description: 'Invoiced revenue net of credit notes, in SEK, recognised on invoice date.', type: 'simple', measure: 'amount_sek', aggregation: 'sum', dimensions: ['country', 'channel'], filters: [{ dimension: 'channel', operator: 'ne', values: ['internal'] }], timeDimension: 'invoiced_at', unit: 'SEK', currency: 'SEK', format: 'currency', owner: 'finance@siax.io', version: 3, status: 'ACTIVE', }; const GOVERNED: GovernedMetric = { spec: REVENUE, governance: { certification: 'certified', business_definition: 'What we actually invoiced, after credits.', technical_definition: "sum(amount_sek) on ds_sales where channel != 'internal', by invoiced_at.", owner: 'finance@siax.io', lineage_refs: ['ds_sales@v42'], quality_status: 'passing', }, }; describe('BI-P028 — a metric that cannot mean nothing', () => { it('accepts the golden metric', () => { expect(() => assertValidMetricSpec(REVENUE)).not.toThrow(); }); const rejections: Array<[string, Partial]> = [ ['no description', { description: '' }], ['no owner', { owner: '' }], ['no time dimension', { timeDimension: '' }], ['a ratio with no denominator', { type: 'ratio' }], ['a denominator on a non-ratio', { denominator: 'orders' }], ['currency format with no currency', { currency: undefined }], ['a currency on a non-currency format', { format: 'integer' }], ['a non-ISO currency', { currency: 'kronor' as string }], ['a name that is not snake_case', { name: 'Net Revenue' }], ['a duplicate dimension', { dimensions: ['country', 'country'] }], [ 'a filter on an undeclared dimension', { filters: [{ dimension: 'segment', operator: 'eq', values: ['x'] }] }, ], ['a value operator with no values', { filters: [{ dimension: 'channel', operator: 'eq', values: [] }] }], [ 'a presence operator with values', { filters: [{ dimension: 'channel', operator: 'is_null', values: ['x'] }] }, ], ['version zero', { version: 0 }], ]; it.each(rejections)('refuses %s', (_label, override) => { expect(() => assertValidMetricSpec({ ...REVENUE, ...override })).toThrowError(SiaxError); }); it('declares the five lifecycle statuses BI-P028 names, in order', () => { expect([...METRIC_STATUSES]).toEqual(['DRAFT', 'REVIEW', 'ACTIVE', 'DEPRECATED', 'RETIRED']); }); }); describe('BI-P028 DoD — no production metric exists only inside a dashboard tool', () => { const lookup = (id: string) => (id === 'm_revenue' ? GOVERNED : undefined); it.each(['inline_expression', 'inline_aggregation', 'inline_sql'])( 'refuses a panel that defines a metric via %s', (field) => { expect(() => assertPanelUsesRegisteredMetric({ panel_id: 'p1', [field]: 'sum(amount)' }, lookup), ).toThrowError(SiaxError); }, ); it('refuses a panel that names an unregistered metric', () => { expect(() => assertPanelUsesRegisteredMetric({ panel_id: 'p1', metric_id: 'm_invented' }, lookup), ).toThrowError(SiaxError); }); it('refuses a panel with no metric at all', () => { expect(() => assertPanelUsesRegisteredMetric({ panel_id: 'p1' }, lookup)).toThrowError(SiaxError); }); it('refuses a panel pointing at a RETIRED metric', () => { const retired = { ...GOVERNED, spec: { ...REVENUE, status: 'RETIRED' as const } }; expect(() => assertPanelUsesRegisteredMetric({ panel_id: 'p1', metric_id: 'm_revenue' }, () => retired), ).toThrowError(SiaxError); }); it('accepts a panel that references a registered, live metric', () => { expect(assertPanelUsesRegisteredMetric({ panel_id: 'p1', metric_id: 'm_revenue' }, lookup).spec.id).toBe( 'm_revenue', ); }); }); describe('BI-P031 DoD — a certified metric cannot be modified in place', () => { /** Every field that changes what the number MEANS. */ const semanticEdits: Array<[string, Partial]> = [ ['the measure', { measure: 'amount_eur' }], ['the aggregation', { aggregation: 'avg' }], ['the type', { type: 'ratio', denominator: 'orders' }], ['the dimensions', { dimensions: ['country'] }], ['the filters', { filters: [] }], ['the time dimension', { timeDimension: 'paid_at' }], ['the unit', { unit: 'EUR' }], ['the currency', { currency: 'EUR' }], ]; it.each(semanticEdits)('refuses an in-place change to %s', (_label, override) => { const proposed = { ...REVENUE, ...override }; let thrown: unknown; try { assertMetricEditAllowed(GOVERNED, proposed); } catch (e) { thrown = e; } expect(thrown, "a certified metric's meaning was changed in place").toBeInstanceOf(SiaxError); expect((thrown as SiaxError).code).toBe('CONFLICT'); expect((thrown as SiaxError).message).toContain('version 4'); }); /** Editorial fields stay editable — forcing a version bump for a typo is how teams route around governance. */ const editorialEdits: Array<[string, Partial]> = [ ['the description', { description: 'Clearer wording.' }], ['the owner', { owner: 'controlling@siax.io' }], ['the lifecycle status', { status: 'DEPRECATED' }], ]; it.each(editorialEdits)('allows an in-place change to %s', (_label, override) => { expect(() => assertMetricEditAllowed(GOVERNED, { ...REVENUE, ...override })).not.toThrow(); }); it('allows any change to an EXPERIMENTAL metric', () => { const experimental = { ...GOVERNED, governance: { ...GOVERNED.governance, certification: 'experimental' as const }, }; expect(() => assertMetricEditAllowed(experimental, { ...REVENUE, measure: 'anything' })).not.toThrow(); }); it('mints a new version that is DRAFT and EXPERIMENTAL, never inheriting the certification', () => { const next = nextMetricVersion( GOVERNED, { measure: 'amount_eur', unit: 'EUR', currency: 'EUR' }, 'switch to EUR', ); expect(next.spec.version).toBe(4); expect(next.spec.status).toBe('DRAFT'); expect(next.governance.certification).toBe('experimental'); expect(next.governance.certified_at).toBeUndefined(); expect(next.change_reason).toBe('switch to EUR'); }); it('requires a change reason for a new version', () => { expect(() => nextMetricVersion(GOVERNED, { measure: 'x' }, ' ')).toThrowError(SiaxError); }); it('refuses to change the metric id under the guise of an edit', () => { expect(() => assertMetricEditAllowed(GOVERNED, { ...REVENUE, id: 'm_other' })).toThrowError(SiaxError); }); it('the meaning hash ignores editorial fields and catches semantic ones', () => { expect(metricMeaningHash({ ...REVENUE, description: 'different', owner: 'x', status: 'DRAFT' })).toBe( metricMeaningHash(REVENUE), ); expect(metricMeaningHash({ ...REVENUE, measure: 'other' })).not.toBe(metricMeaningHash(REVENUE)); // Filter ORDER is not meaning; filter CONTENT is. expect( metricMeaningHash({ ...REVENUE, filters: [ { dimension: 'country', operator: 'eq', values: ['SE', 'NO'] }, { dimension: 'channel', operator: 'ne', values: ['internal'] }, ], }), ).not.toBe(metricMeaningHash(REVENUE)); }); it('certification is an act with an actor and a time, and needs both definitions', () => { const draft: GovernedMetric = { spec: { ...REVENUE, status: 'DRAFT' }, governance: { ...GOVERNED.governance, certification: 'experimental' }, }; expect(() => certifyMetric(draft, 'u_1', '2026-09-07T00:00:00Z')).toThrowError(SiaxError); const active = { ...draft, spec: { ...draft.spec, status: 'ACTIVE' as const } }; const noDefs = { ...active, governance: { ...active.governance, technical_definition: '' } }; expect(() => certifyMetric(noDefs, 'u_1', '2026-09-07T00:00:00Z')).toThrowError(SiaxError); const certified = certifyMetric(active, 'u_1', '2026-09-07T00:00:00Z'); expect(certified.governance.certification).toBe('certified'); expect(certified.governance.certified_by).toBe('u_1'); }); }); // ── BI-P030 ──────────────────────────────────────────────────────────────── function model(version = 1): Omit { return { model_id: 'sm_sales', version, tenant_id: 't_alpha', datasets: ['ds_sales'], relationships: [], metrics: [REVENUE], dimensions: [ { id: 'd_country', name: 'country', dataset_id: 'ds_sales', kind: 'categorical' }, { id: 'd_channel', name: 'channel', dataset_id: 'ds_sales', kind: 'categorical' }, { id: 'd_invoiced', name: 'invoiced_at', dataset_id: 'ds_sales', kind: 'time' }, ], valid_from: '2026-09-01T00:00:00Z', status: 'draft', }; } describe('BI-P030 — the registry is engine-neutral and its versions are immutable', () => { it('refuses a model that names an implementation engine anywhere in it', async () => { for (const smuggled of [ { model_id: 'sm_metricflow_sales' }, { datasets: ['clickhouse_sales'] }, { metrics: [{ ...REVENUE, description: 'computed by MetricFlow' }] }, ]) { expect(() => assertEngineNeutral({ ...model(), ...smuggled } as Omit), ).toThrowError(SiaxError); } }); it('refuses a metric naming a dimension the model does not define', async () => { const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); await expect( registry.publish({ ...model(), metrics: [{ ...REVENUE, dimensions: ['country', 'segment'] }] }), ).rejects.toBeInstanceOf(SiaxError); }); it('refuses a metric whose time dimension is not declared as a time dimension', async () => { const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); await expect( registry.publish({ ...model(), metrics: [{ ...REVENUE, timeDimension: 'country' }] }), ).rejects.toBeInstanceOf(SiaxError); }); it('refuses to republish an existing version — a pinned dashboard must mean the same thing tomorrow', async () => { const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); await registry.publish(model(1)); await expect(registry.publish(model(1))).rejects.toBeInstanceOf(SiaxError); }); it('supersedes the previous active version on activation', async () => { const store = new InMemorySemanticModelStore(); const registry = new SemanticModelRegistry(store); await registry.publish(model(1)); await registry.publish(model(2)); await registry.activate('t_alpha', 'sm_sales', 1, '2026-09-01T00:00:00Z'); await registry.activate('t_alpha', 'sm_sales', 2, '2026-09-05T00:00:00Z'); expect((await store.get('t_alpha', 'sm_sales', 1))?.status).toBe('superseded'); expect((await store.active('t_alpha', 'sm_sales'))?.version).toBe(2); }); it('the hash covers meaning and ignores activation', () => { const a = semanticModelHash(model(1)); const b = semanticModelHash({ ...model(1), valid_from: '2030-01-01T00:00:00Z', status: 'active', } as never); expect(a).toBe(b); const c = semanticModelHash({ ...model(1), metrics: [{ ...REVENUE, version: 4 }] }); expect(c).not.toBe(a); }); it('requires a verified tenant', async () => { const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); await expect(registry.publish({ ...model(), tenant_id: '' })).rejects.toBeInstanceOf(SiaxError); }); }); describe('BI-P030 DoD — the semantic engine can be replaced without breaking metric IDs', () => { /** * A SECOND provider with DIFFERENT naming rules, so the test cannot pass * trivially for a registry that only ever had one provider. Cube-shaped: * metric names are `cube.measure`, which is exactly the kind of difference * that would break a consumer if canonical ids were not the contract. */ class CubeShapedProvider implements SiaxSemanticProvider { readonly provider = 'cube-shaped'; buildManifest(m: SemanticModelVersion) { return { cubes: m.datasets.map((d) => ({ name: d, measures: m.metrics.map((x) => x.measure) })) }; } compile(spec: ReturnType, m: SemanticModelVersion) { const bindings = this.providerBindings(m); return { provider: this.provider, query: { measures: spec.metrics.map((x) => bindings[x.metric_id]) }, canonical_metric_ids: spec.metrics.map((x) => x.metric_id).sort(), }; } providerBindings(m: SemanticModelVersion): Record { return Object.fromEntries(m.metrics.map((x) => [x.id, `ds_sales.${x.measure}`])); } } it('both providers preserve the canonical ids while naming them differently', async () => { const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); const published = await registry.publish(model(1)); const resolution = queryResolutionFrom({ dataset_id: 'ds_sales', metrics: [ { name: 'net_revenue', metric_id: 'm_revenue', event_metric_key: 'amount_sek', aggregation: 'sum', allowed_dimensions: ['country', 'channel'], }, ], dimensions: [{ name: 'country', dimension_id: 'd_country', field_name: 'country_code' }], semantic_model_version: 1, }); const spec = compileRequestToSemantic( resolveAnalyticsQuery( parseAnalyticsQueryRequest({ dataset: 'ds_sales', metrics: ['net_revenue'], dimensions: ['country'], }), resolution, ), ); const mf = new MetricFlowProvider(); const cube = new CubeShapedProvider(); expect(mf.compile(spec, published).canonical_metric_ids).toEqual(['m_revenue']); expect(cube.compile(spec, published).canonical_metric_ids).toEqual(['m_revenue']); expect(await registry.canonicalMetricIds('t_alpha', 'sm_sales', 1)).toEqual(['m_revenue']); // ...and the providers genuinely differ, so the equality above is not // trivially true. expect(mf.providerBindings(published)).not.toEqual(cube.providerBindings(published)); expect(cube.providerBindings(published).m_revenue).toBe('ds_sales.amount_sek'); }); it('the MetricFlow manifest is DERIVED, never persisted into the registry', async () => { const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); const published = await registry.publish(model(1)); const manifest = new MetricFlowProvider().buildManifest(published); expect(manifest.metrics[0].name).toBe('m_revenue'); expect(manifest._generated_from.hash).toBe(published.hash); // Two independent reasons a registry row cannot become a provider manifest. // The vocabulary scan catches the generator stamp... expect(() => assertEngineNeutral({ ...published, ...(manifest as object) } as never)).toThrowError( SiaxError, ); // ...and the structural check catches ANY extra field, which is what holds // for a manifest that avoids the word. The first version of this test // asserted only the vocabulary scan and passed for the wrong reason: // buildManifest did not mention "metricflow" anywhere, so nothing fired. expect(() => assertOnlySemanticModelFields({ ...published, semantic_models: [] })).toThrowError( SiaxError, ); }); it("publish refuses a model carrying any field outside BI-P030's list", async () => { const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); await expect(registry.publish({ ...model(1), compiled_sql: 'SELECT 1' } as never)).rejects.toBeInstanceOf( SiaxError, ); }); it('refuses to compile a metric that is not in the model version', async () => { const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); const published = await registry.publish(model(1)); expect(() => new MetricFlowProvider().compile( { dataset_id: 'ds_sales', metrics: [{ metric_id: 'm_ghost', name: 'ghost' }], groupBy: [], where: [], orderBy: [], limit: 10, format: 'json', }, published, ), ).toThrowError(SiaxError); }); it('puts no SQL predicate text into the compiled provider query', async () => { const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); const published = await registry.publish(model(1)); const compiled = new MetricFlowProvider().compile( { dataset_id: 'ds_sales', metrics: [{ metric_id: 'm_revenue', name: 'net_revenue' }], groupBy: [], where: [{ dimension_id: 'd_country', name: 'country', operator: 'eq', values: ["SE' OR '1'='1"] }], orderBy: [], limit: 10, format: 'json', }, published, ); const serialized = JSON.stringify(compiled.query); expect(serialized).toContain("SE' OR '1'='1"); // survives as DATA expect(serialized).not.toMatch(/\bWHERE\b|\bSELECT\b/); }); }); describe('BI-P029 — the provider states what it has NOT proven', () => { it('reports itself unverified, and names the open surfaces', () => { const readiness = new MetricFlowProvider().providerReadiness(); expect(readiness.verified).toBe(false); expect(readiness.open.join(' ')).toContain('NOT been executed'); expect(readiness.open.join(' ')).toContain('Superset'); expect(readiness.open.join(' ')).toContain('AI Analyst'); expect(readiness.proven.length).toBeGreaterThan(0); }); });