Files
0pt/tests/analytics/metric-governance.test.ts
T

484 lines
20 KiB
TypeScript

/**
* tests/analytics/metric-governance.test.ts
* ==========================================
* BI-P028 (metric domain model), BI-P030 (semantic model registry),
* BI-P031 (metric governance), BI-P029 (semantic provider — the parts not
* needing a Python runtime).
*
* Each prompt's DoD is turned into something that can FAIL:
*
* BI-P028 "No production metric exists only inside a dashboard tool."
* A dashboard panel carrying an inline aggregation / expression /
* SQL is REFUSED, and so is one naming an unregistered metric.
*
* BI-P030 "MetricFlow can be replaced without breaking canonical SIAX
* metric IDs." Two providers with different naming rules compile
* the same registry version; the canonical id set must be
* identical and the provider BINDINGS must differ — otherwise the
* test would pass for a registry that simply has one provider.
*
* BI-P031 "Certified metric cannot be modified in-place; new version
* required." Every semantic field is mutated one at a time and
* each must be refused; every editorial field is mutated and each
* must be allowed.
*/
import { describe, expect, it } from 'vitest';
import { SiaxError } from '@/src/platform/errors';
import {
assertMetricEditAllowed,
assertPanelUsesRegisteredMetric,
assertValidMetricSpec,
certifyMetric,
metricMeaningHash,
nextMetricVersion,
METRIC_STATUSES,
type GovernedMetric,
type MetricSpec,
} from '@/src/analytics/metrics/metric-model';
import {
assertEngineNeutral,
assertOnlySemanticModelFields,
InMemorySemanticModelStore,
SemanticModelRegistry,
semanticModelHash,
type SemanticModelVersion,
} from '@/src/analytics/metrics/semantic-registry';
import { MetricFlowProvider, type SiaxSemanticProvider } from '@/src/analytics/metrics/semantic-provider';
import { compileRequestToSemantic } from '@/src/analytics/contract/compile-semantic';
import {
parseAnalyticsQueryRequest,
queryResolutionFrom,
resolveAnalyticsQuery,
} from '@/src/analytics/contract/query-request';
const REVENUE: MetricSpec = {
id: 'm_revenue',
name: 'net_revenue',
description: 'Invoiced revenue net of credit notes, in SEK, recognised on invoice date.',
type: 'simple',
measure: 'amount_sek',
aggregation: 'sum',
dimensions: ['country', 'channel'],
filters: [{ dimension: 'channel', operator: 'ne', values: ['internal'] }],
timeDimension: 'invoiced_at',
unit: 'SEK',
currency: 'SEK',
format: 'currency',
owner: 'finance@siax.io',
version: 3,
status: 'ACTIVE',
};
const GOVERNED: GovernedMetric = {
spec: REVENUE,
governance: {
certification: 'certified',
business_definition: 'What we actually invoiced, after credits.',
technical_definition: "sum(amount_sek) on ds_sales where channel != 'internal', by invoiced_at.",
owner: 'finance@siax.io',
lineage_refs: ['ds_sales@v42'],
quality_status: 'passing',
},
};
describe('BI-P028 — a metric that cannot mean nothing', () => {
it('accepts the golden metric', () => {
expect(() => assertValidMetricSpec(REVENUE)).not.toThrow();
});
const rejections: Array<[string, Partial<MetricSpec>]> = [
['no description', { description: '' }],
['no owner', { owner: '' }],
['no time dimension', { timeDimension: '' }],
['a ratio with no denominator', { type: 'ratio' }],
['a denominator on a non-ratio', { denominator: 'orders' }],
['currency format with no currency', { currency: undefined }],
['a currency on a non-currency format', { format: 'integer' }],
['a non-ISO currency', { currency: 'kronor' as string }],
['a name that is not snake_case', { name: 'Net Revenue' }],
['a duplicate dimension', { dimensions: ['country', 'country'] }],
[
'a filter on an undeclared dimension',
{ filters: [{ dimension: 'segment', operator: 'eq', values: ['x'] }] },
],
['a value operator with no values', { filters: [{ dimension: 'channel', operator: 'eq', values: [] }] }],
[
'a presence operator with values',
{ filters: [{ dimension: 'channel', operator: 'is_null', values: ['x'] }] },
],
['version zero', { version: 0 }],
];
it.each(rejections)('refuses %s', (_label, override) => {
expect(() => assertValidMetricSpec({ ...REVENUE, ...override })).toThrowError(SiaxError);
});
it('declares the five lifecycle statuses BI-P028 names, in order', () => {
expect([...METRIC_STATUSES]).toEqual(['DRAFT', 'REVIEW', 'ACTIVE', 'DEPRECATED', 'RETIRED']);
});
});
describe('BI-P028 DoD — no production metric exists only inside a dashboard tool', () => {
const lookup = (id: string) => (id === 'm_revenue' ? GOVERNED : undefined);
it.each(['inline_expression', 'inline_aggregation', 'inline_sql'])(
'refuses a panel that defines a metric via %s',
(field) => {
expect(() =>
assertPanelUsesRegisteredMetric({ panel_id: 'p1', [field]: 'sum(amount)' }, lookup),
).toThrowError(SiaxError);
},
);
it('refuses a panel that names an unregistered metric', () => {
expect(() =>
assertPanelUsesRegisteredMetric({ panel_id: 'p1', metric_id: 'm_invented' }, lookup),
).toThrowError(SiaxError);
});
it('refuses a panel with no metric at all', () => {
expect(() => assertPanelUsesRegisteredMetric({ panel_id: 'p1' }, lookup)).toThrowError(SiaxError);
});
it('refuses a panel pointing at a RETIRED metric', () => {
const retired = { ...GOVERNED, spec: { ...REVENUE, status: 'RETIRED' as const } };
expect(() =>
assertPanelUsesRegisteredMetric({ panel_id: 'p1', metric_id: 'm_revenue' }, () => retired),
).toThrowError(SiaxError);
});
it('accepts a panel that references a registered, live metric', () => {
expect(assertPanelUsesRegisteredMetric({ panel_id: 'p1', metric_id: 'm_revenue' }, lookup).spec.id).toBe(
'm_revenue',
);
});
});
describe('BI-P031 DoD — a certified metric cannot be modified in place', () => {
/** Every field that changes what the number MEANS. */
const semanticEdits: Array<[string, Partial<MetricSpec>]> = [
['the measure', { measure: 'amount_eur' }],
['the aggregation', { aggregation: 'avg' }],
['the type', { type: 'ratio', denominator: 'orders' }],
['the dimensions', { dimensions: ['country'] }],
['the filters', { filters: [] }],
['the time dimension', { timeDimension: 'paid_at' }],
['the unit', { unit: 'EUR' }],
['the currency', { currency: 'EUR' }],
];
it.each(semanticEdits)('refuses an in-place change to %s', (_label, override) => {
const proposed = { ...REVENUE, ...override };
let thrown: unknown;
try {
assertMetricEditAllowed(GOVERNED, proposed);
} catch (e) {
thrown = e;
}
expect(thrown, "a certified metric's meaning was changed in place").toBeInstanceOf(SiaxError);
expect((thrown as SiaxError).code).toBe('CONFLICT');
expect((thrown as SiaxError).message).toContain('version 4');
});
/** Editorial fields stay editable — forcing a version bump for a typo is how teams route around governance. */
const editorialEdits: Array<[string, Partial<MetricSpec>]> = [
['the description', { description: 'Clearer wording.' }],
['the owner', { owner: 'controlling@siax.io' }],
['the lifecycle status', { status: 'DEPRECATED' }],
];
it.each(editorialEdits)('allows an in-place change to %s', (_label, override) => {
expect(() => assertMetricEditAllowed(GOVERNED, { ...REVENUE, ...override })).not.toThrow();
});
it('allows any change to an EXPERIMENTAL metric', () => {
const experimental = {
...GOVERNED,
governance: { ...GOVERNED.governance, certification: 'experimental' as const },
};
expect(() => assertMetricEditAllowed(experimental, { ...REVENUE, measure: 'anything' })).not.toThrow();
});
it('mints a new version that is DRAFT and EXPERIMENTAL, never inheriting the certification', () => {
const next = nextMetricVersion(
GOVERNED,
{ measure: 'amount_eur', unit: 'EUR', currency: 'EUR' },
'switch to EUR',
);
expect(next.spec.version).toBe(4);
expect(next.spec.status).toBe('DRAFT');
expect(next.governance.certification).toBe('experimental');
expect(next.governance.certified_at).toBeUndefined();
expect(next.change_reason).toBe('switch to EUR');
});
it('requires a change reason for a new version', () => {
expect(() => nextMetricVersion(GOVERNED, { measure: 'x' }, ' ')).toThrowError(SiaxError);
});
it('refuses to change the metric id under the guise of an edit', () => {
expect(() => assertMetricEditAllowed(GOVERNED, { ...REVENUE, id: 'm_other' })).toThrowError(SiaxError);
});
it('the meaning hash ignores editorial fields and catches semantic ones', () => {
expect(metricMeaningHash({ ...REVENUE, description: 'different', owner: 'x', status: 'DRAFT' })).toBe(
metricMeaningHash(REVENUE),
);
expect(metricMeaningHash({ ...REVENUE, measure: 'other' })).not.toBe(metricMeaningHash(REVENUE));
// Filter ORDER is not meaning; filter CONTENT is.
expect(
metricMeaningHash({
...REVENUE,
filters: [
{ dimension: 'country', operator: 'eq', values: ['SE', 'NO'] },
{ dimension: 'channel', operator: 'ne', values: ['internal'] },
],
}),
).not.toBe(metricMeaningHash(REVENUE));
});
it('certification is an act with an actor and a time, and needs both definitions', () => {
const draft: GovernedMetric = {
spec: { ...REVENUE, status: 'DRAFT' },
governance: { ...GOVERNED.governance, certification: 'experimental' },
};
expect(() => certifyMetric(draft, 'u_1', '2026-09-07T00:00:00Z')).toThrowError(SiaxError);
const active = { ...draft, spec: { ...draft.spec, status: 'ACTIVE' as const } };
const noDefs = { ...active, governance: { ...active.governance, technical_definition: '' } };
expect(() => certifyMetric(noDefs, 'u_1', '2026-09-07T00:00:00Z')).toThrowError(SiaxError);
const certified = certifyMetric(active, 'u_1', '2026-09-07T00:00:00Z');
expect(certified.governance.certification).toBe('certified');
expect(certified.governance.certified_by).toBe('u_1');
});
});
// ── BI-P030 ────────────────────────────────────────────────────────────────
function model(version = 1): Omit<SemanticModelVersion, 'hash'> {
return {
model_id: 'sm_sales',
version,
tenant_id: 't_alpha',
datasets: ['ds_sales'],
relationships: [],
metrics: [REVENUE],
dimensions: [
{ id: 'd_country', name: 'country', dataset_id: 'ds_sales', kind: 'categorical' },
{ id: 'd_channel', name: 'channel', dataset_id: 'ds_sales', kind: 'categorical' },
{ id: 'd_invoiced', name: 'invoiced_at', dataset_id: 'ds_sales', kind: 'time' },
],
valid_from: '2026-09-01T00:00:00Z',
status: 'draft',
};
}
describe('BI-P030 — the registry is engine-neutral and its versions are immutable', () => {
it('refuses a model that names an implementation engine anywhere in it', async () => {
for (const smuggled of [
{ model_id: 'sm_metricflow_sales' },
{ datasets: ['clickhouse_sales'] },
{ metrics: [{ ...REVENUE, description: 'computed by MetricFlow' }] },
]) {
expect(() =>
assertEngineNeutral({ ...model(), ...smuggled } as Omit<SemanticModelVersion, 'hash'>),
).toThrowError(SiaxError);
}
});
it('refuses a metric naming a dimension the model does not define', async () => {
const registry = new SemanticModelRegistry(new InMemorySemanticModelStore());
await expect(
registry.publish({ ...model(), metrics: [{ ...REVENUE, dimensions: ['country', 'segment'] }] }),
).rejects.toBeInstanceOf(SiaxError);
});
it('refuses a metric whose time dimension is not declared as a time dimension', async () => {
const registry = new SemanticModelRegistry(new InMemorySemanticModelStore());
await expect(
registry.publish({ ...model(), metrics: [{ ...REVENUE, timeDimension: 'country' }] }),
).rejects.toBeInstanceOf(SiaxError);
});
it('refuses to republish an existing version — a pinned dashboard must mean the same thing tomorrow', async () => {
const registry = new SemanticModelRegistry(new InMemorySemanticModelStore());
await registry.publish(model(1));
await expect(registry.publish(model(1))).rejects.toBeInstanceOf(SiaxError);
});
it('supersedes the previous active version on activation', async () => {
const store = new InMemorySemanticModelStore();
const registry = new SemanticModelRegistry(store);
await registry.publish(model(1));
await registry.publish(model(2));
await registry.activate('t_alpha', 'sm_sales', 1, '2026-09-01T00:00:00Z');
await registry.activate('t_alpha', 'sm_sales', 2, '2026-09-05T00:00:00Z');
expect((await store.get('t_alpha', 'sm_sales', 1))?.status).toBe('superseded');
expect((await store.active('t_alpha', 'sm_sales'))?.version).toBe(2);
});
it('the hash covers meaning and ignores activation', () => {
const a = semanticModelHash(model(1));
const b = semanticModelHash({
...model(1),
valid_from: '2030-01-01T00:00:00Z',
status: 'active',
} as never);
expect(a).toBe(b);
const c = semanticModelHash({ ...model(1), metrics: [{ ...REVENUE, version: 4 }] });
expect(c).not.toBe(a);
});
it('requires a verified tenant', async () => {
const registry = new SemanticModelRegistry(new InMemorySemanticModelStore());
await expect(registry.publish({ ...model(), tenant_id: '' })).rejects.toBeInstanceOf(SiaxError);
});
});
describe('BI-P030 DoD — the semantic engine can be replaced without breaking metric IDs', () => {
/**
* A SECOND provider with DIFFERENT naming rules, so the test cannot pass
* trivially for a registry that only ever had one provider. Cube-shaped:
* metric names are `cube.measure`, which is exactly the kind of difference
* that would break a consumer if canonical ids were not the contract.
*/
class CubeShapedProvider implements SiaxSemanticProvider {
readonly provider = 'cube-shaped';
buildManifest(m: SemanticModelVersion) {
return { cubes: m.datasets.map((d) => ({ name: d, measures: m.metrics.map((x) => x.measure) })) };
}
compile(spec: ReturnType<typeof compileRequestToSemantic>, m: SemanticModelVersion) {
const bindings = this.providerBindings(m);
return {
provider: this.provider,
query: { measures: spec.metrics.map((x) => bindings[x.metric_id]) },
canonical_metric_ids: spec.metrics.map((x) => x.metric_id).sort(),
};
}
providerBindings(m: SemanticModelVersion): Record<string, string> {
return Object.fromEntries(m.metrics.map((x) => [x.id, `ds_sales.${x.measure}`]));
}
}
it('both providers preserve the canonical ids while naming them differently', async () => {
const registry = new SemanticModelRegistry(new InMemorySemanticModelStore());
const published = await registry.publish(model(1));
const resolution = queryResolutionFrom({
dataset_id: 'ds_sales',
metrics: [
{
name: 'net_revenue',
metric_id: 'm_revenue',
event_metric_key: 'amount_sek',
aggregation: 'sum',
allowed_dimensions: ['country', 'channel'],
},
],
dimensions: [{ name: 'country', dimension_id: 'd_country', field_name: 'country_code' }],
semantic_model_version: 1,
});
const spec = compileRequestToSemantic(
resolveAnalyticsQuery(
parseAnalyticsQueryRequest({
dataset: 'ds_sales',
metrics: ['net_revenue'],
dimensions: ['country'],
}),
resolution,
),
);
const mf = new MetricFlowProvider();
const cube = new CubeShapedProvider();
expect(mf.compile(spec, published).canonical_metric_ids).toEqual(['m_revenue']);
expect(cube.compile(spec, published).canonical_metric_ids).toEqual(['m_revenue']);
expect(await registry.canonicalMetricIds('t_alpha', 'sm_sales', 1)).toEqual(['m_revenue']);
// ...and the providers genuinely differ, so the equality above is not
// trivially true.
expect(mf.providerBindings(published)).not.toEqual(cube.providerBindings(published));
expect(cube.providerBindings(published).m_revenue).toBe('ds_sales.amount_sek');
});
it('the MetricFlow manifest is DERIVED, never persisted into the registry', async () => {
const registry = new SemanticModelRegistry(new InMemorySemanticModelStore());
const published = await registry.publish(model(1));
const manifest = new MetricFlowProvider().buildManifest(published);
expect(manifest.metrics[0].name).toBe('m_revenue');
expect(manifest._generated_from.hash).toBe(published.hash);
// Two independent reasons a registry row cannot become a provider manifest.
// The vocabulary scan catches the generator stamp...
expect(() => assertEngineNeutral({ ...published, ...(manifest as object) } as never)).toThrowError(
SiaxError,
);
// ...and the structural check catches ANY extra field, which is what holds
// for a manifest that avoids the word. The first version of this test
// asserted only the vocabulary scan and passed for the wrong reason:
// buildManifest did not mention "metricflow" anywhere, so nothing fired.
expect(() => assertOnlySemanticModelFields({ ...published, semantic_models: [] })).toThrowError(
SiaxError,
);
});
it("publish refuses a model carrying any field outside BI-P030's list", async () => {
const registry = new SemanticModelRegistry(new InMemorySemanticModelStore());
await expect(registry.publish({ ...model(1), compiled_sql: 'SELECT 1' } as never)).rejects.toBeInstanceOf(
SiaxError,
);
});
it('refuses to compile a metric that is not in the model version', async () => {
const registry = new SemanticModelRegistry(new InMemorySemanticModelStore());
const published = await registry.publish(model(1));
expect(() =>
new MetricFlowProvider().compile(
{
dataset_id: 'ds_sales',
metrics: [{ metric_id: 'm_ghost', name: 'ghost' }],
groupBy: [],
where: [],
orderBy: [],
limit: 10,
format: 'json',
},
published,
),
).toThrowError(SiaxError);
});
it('puts no SQL predicate text into the compiled provider query', async () => {
const registry = new SemanticModelRegistry(new InMemorySemanticModelStore());
const published = await registry.publish(model(1));
const compiled = new MetricFlowProvider().compile(
{
dataset_id: 'ds_sales',
metrics: [{ metric_id: 'm_revenue', name: 'net_revenue' }],
groupBy: [],
where: [{ dimension_id: 'd_country', name: 'country', operator: 'eq', values: ["SE' OR '1'='1"] }],
orderBy: [],
limit: 10,
format: 'json',
},
published,
);
const serialized = JSON.stringify(compiled.query);
expect(serialized).toContain("SE' OR '1'='1"); // survives as DATA
expect(serialized).not.toMatch(/\bWHERE\b|\bSELECT\b/);
});
});
describe('BI-P029 — the provider states what it has NOT proven', () => {
it('reports itself unverified, and names the open surfaces', () => {
const readiness = new MetricFlowProvider().providerReadiness();
expect(readiness.verified).toBe(false);
expect(readiness.open.join(' ')).toContain('NOT been executed');
expect(readiness.open.join(' ')).toContain('Superset');
expect(readiness.open.join(' ')).toContain('AI Analyst');
expect(readiness.proven.length).toBeGreaterThan(0);
});
});