/** * tests/analytics/quality-and-freshness.test.ts * ============================================== * BI-P025 (data quality provider) and BI-P026 (data freshness). * * Both DoDs are about a FALSE GREEN, so both are tested by constructing the * false green and requiring a failure: * * BI-P025 A quality gate that passes when nothing ran. A `not_null` check * over an EMPTY table finds zero violations — the single most * common false green in data quality. Both are asserted to FAIL. * * BI-P026 UNKNOWN age quietly rendering as current, and a result released * with no freshness envelope at all. Both are refused. */ import { describe, expect, it } from 'vitest'; import { SiaxError } from '@/src/platform/errors'; import { DbtQualityProvider, GreatExpectationsQualityProvider, NativeQualityProvider, QUALITY_RULES, type QualityQueryRunner, type QualityRuleSpec, type QualityVerdict, } from '@/src/analytics/quality/quality-provider'; import { InMemoryQuarantineStore, QualityGate } from '@/src/analytics/quality/quarantine'; import { assertFreshnessDisclosed, computeFreshness, FRESHNESS_STATES, isPresentableAsCurrent, NON_CURRENT_STATES, } from '@/src/analytics/quality/freshness'; function runner( violations: number, evaluated: number, ): QualityQueryRunner & { lastExpression?: string; lastParams?: Record } { const r: QualityQueryRunner & { lastExpression?: string; lastParams?: Record } = { engine: 'clickhouse', async count(args) { r.lastExpression = args.violationExpression; r.lastParams = args.params; return { violations, evaluated }; }, }; return r; } const RULE: QualityRuleSpec = { rule_id: 'q_not_null_customer', tenant_id: 't_alpha', dataset_id: 'ds_sales', rule: 'not_null', field: 'customer', severity: 'block', params: {}, }; describe('BI-P025 — the native provider EXECUTES, and says so', () => { it('declares every rule BI-P025 names', () => { expect([...QUALITY_RULES]).toEqual([ 'not_null', 'unique', 'range', 'referential', 'freshness', 'schema', 'distribution', 'custom', ]); }); it('passes a clean dataset', async () => { const verdict = await new NativeQualityProvider(runner(0, 1000)).evaluate(RULE); expect(verdict.passed).toBe(true); expect(verdict.provider_kind).toBe('executing'); }); it('FAILS an empty dataset instead of reporting zero violations', async () => { // The false green this whole file exists for: a not-null check over zero // rows finds zero violations and looks healthy while the pipeline is dead. const verdict = await new NativeQualityProvider(runner(0, 0)).evaluate(RULE); expect(verdict.passed).toBe(false); expect(verdict.detail).toContain('EMPTY'); }); it('fails a dataset with violations', async () => { const verdict = await new NativeQualityProvider(runner(7, 1000)).evaluate(RULE); expect(verdict.passed).toBe(false); expect(verdict.violation_count).toBe(7); }); it('binds rule parameters instead of splicing them into the expression', async () => { const r = runner(0, 10); await new NativeQualityProvider(r).evaluate({ ...RULE, rule: 'distribution', field: 'channel', params: { accepted_values: ['web', "retail'; DROP TABLE x; --"] }, }); expect(r.lastExpression).not.toContain('DROP'); expect(Object.values(r.lastParams ?? {})).toContain("retail'; DROP TABLE x; --"); }); it('refuses a field name that is not an identifier', async () => { const p = new NativeQualityProvider(runner(0, 10)); await expect(p.evaluate({ ...RULE, field: 'customer; DROP TABLE x' })).rejects.toBeInstanceOf(SiaxError); }); it('refuses an incomplete rule rather than evaluating a meaningless one', async () => { const p = new NativeQualityProvider(runner(0, 10)); await expect(p.evaluate({ ...RULE, rule: 'range', params: {} })).rejects.toBeInstanceOf(SiaxError); await expect(p.evaluate({ ...RULE, rule: 'freshness', params: {} })).rejects.toBeInstanceOf(SiaxError); await expect(p.evaluate({ ...RULE, rule: 'distribution', params: {} })).rejects.toBeInstanceOf(SiaxError); await expect(p.evaluate({ ...RULE, rule: 'custom', params: {} })).rejects.toBeInstanceOf(SiaxError); }); it('says CAPABILITY_UNAVAILABLE for rules it cannot evaluate, rather than passing them', async () => { const p = new NativeQualityProvider(runner(0, 10)); for (const rule of ['schema', 'referential'] as const) { await expect(p.evaluate({ ...RULE, rule })).rejects.toBeInstanceOf(SiaxError); expect(p.supports(rule)).toBe(false); } }); }); describe('BI-P025 — the ingesting providers do not pretend to execute', () => { it('dbt and GE declare themselves as ingesting', async () => { expect(new DbtQualityProvider(new Map()).providerKind).toBe('ingesting'); expect(new GreatExpectationsQualityProvider(new Map()).providerKind).toBe('ingesting'); }); it('a MISSING dbt result is a failure, not a pass', async () => { // Treating "no result" as "passed" is how a gate stops gating the moment // the test stops running. const verdict = await new DbtQualityProvider(new Map()).evaluate(RULE); expect(verdict.passed).toBe(false); expect(verdict.detail).toContain('no dbt result'); }); it.each(['error', 'warn', 'skipped'] as const)('dbt status %s is NOT a pass', async (status) => { const results = new Map([[RULE.rule_id, { unique_id: 't1', status, failures: 0 }]]); const verdict = await new DbtQualityProvider(results).evaluate(RULE); expect(verdict.passed).toBe(false); }); it('dbt pass is a pass', async () => { const results = new Map([[RULE.rule_id, { unique_id: 't1', status: 'pass' as const, failures: 0 }]]); expect((await new DbtQualityProvider(results).evaluate(RULE)).passed).toBe(true); }); it('a missing GE suite is a failure', async () => { expect((await new GreatExpectationsQualityProvider(new Map()).evaluate(RULE)).passed).toBe(false); }); it("a GE suite's unexpected counts are carried through", async () => { const suites = new Map([ [ RULE.rule_id, { expectation_suite_name: 'sales_suite', success: false, results: [ { expectation_config: { expectation_type: 'expect_column_values_to_not_be_null' }, success: false, result: { unexpected_count: 12, element_count: 1000 }, }, ], }, ], ]); const verdict = await new GreatExpectationsQualityProvider(suites).evaluate(RULE); expect(verdict.passed).toBe(false); expect(verdict.violation_count).toBe(12); expect(verdict.evaluated_rows).toBe(1000); }); }); describe('BI-P025 DoD — a failed critical gate QUARANTINES the serving projection', () => { function verdict(overrides: Partial = {}): QualityVerdict { return { rule_id: 'q1', dataset_id: 'ds_sales', tenant_id: 't_alpha', rule: 'not_null', severity: 'block', passed: true, violation_count: 0, evaluated_rows: 100, produced_by: 'native:clickhouse', provider_kind: 'executing', evaluated_at: new Date().toISOString(), ...overrides, }; } it('a blocking failure quarantines, and the READ is then REFUSED', async () => { const gate = new QualityGate(new InMemoryQuarantineStore()); // Before: readable. await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).resolves.toBeUndefined(); const outcome = await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: false, severity: 'block' })]); expect(outcome.quarantined).toBe(true); expect(outcome.passed).toBe(false); // After: refused. Not annotated — refused. await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).rejects.toBeInstanceOf(SiaxError); }); it('a WARN failure does not quarantine, but is surfaced', async () => { const gate = new QualityGate(new InMemoryQuarantineStore()); const outcome = await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: false, severity: 'warn' })]); expect(outcome.quarantined).toBe(false); expect(outcome.warnings).toHaveLength(1); await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).resolves.toBeUndefined(); }); it("a gate over ZERO verdicts QUARANTINES — 'no rules ran' is not 'everything passed'", async () => { // The same shape as l0re's silently-skipped tests and api0's // `trivy --exit-code 0`: a check that produced nothing reporting green. const gate = new QualityGate(new InMemoryQuarantineStore()); const outcome = await gate.apply('t_alpha', 'ds_sales', []); expect(outcome.passed).toBe(false); expect(outcome.quarantined).toBe(true); await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).rejects.toBeInstanceOf(SiaxError); }); it('a later passing run does NOT auto-release a quarantine', async () => { const gate = new QualityGate(new InMemoryQuarantineStore()); await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: false })]); await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: true })]); // A flapping rule would otherwise quarantine and un-quarantine the dataset // between two reads of the same dashboard. await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).rejects.toBeInstanceOf(SiaxError); }); it('release requires a principal and a reason', async () => { const gate = new QualityGate(new InMemoryQuarantineStore()); await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: false })]); await expect(gate.release('t_alpha', 'ds_sales', '', 'x')).rejects.toBeInstanceOf(SiaxError); await expect(gate.release('t_alpha', 'ds_sales', 'u_1', ' ')).rejects.toBeInstanceOf(SiaxError); const released = await gate.release('t_alpha', 'ds_sales', 'u_1', 'upstream backfilled'); expect(released.released_by).toBe('u_1'); await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).resolves.toBeUndefined(); }); it("quarantine is tenant-scoped: one tenant's failure does not block another", async () => { const gate = new QualityGate(new InMemoryQuarantineStore()); await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: false })]); await expect(gate.assertProjectionReadable('t_beta', 'ds_sales')).resolves.toBeUndefined(); }); it('refuses a verdict belonging to another tenant or dataset', async () => { const gate = new QualityGate(new InMemoryQuarantineStore()); await expect( gate.apply('t_alpha', 'ds_sales', [verdict({ tenant_id: 't_beta' })]), ).rejects.toBeInstanceOf(SiaxError); await expect( gate.apply('t_alpha', 'ds_sales', [verdict({ dataset_id: 'ds_other' })]), ).rejects.toBeInstanceOf(SiaxError); }); }); describe('BI-P026 — five states, and UNKNOWN is not FRESH', () => { const HOUR = 3_600_000; const NOW = Date.parse('2026-09-07T12:00:00Z'); const policy = { expectedIntervalMs: HOUR }; it('declares the five states BI-P026 names', () => { expect([...FRESHNESS_STATES]).toEqual(['FRESH', 'DELAYED', 'STALE', 'FAILED', 'UNKNOWN']); }); it('only FRESH may be presented as current', () => { expect(isPresentableAsCurrent('FRESH')).toBe(true); for (const s of NON_CURRENT_STATES) expect(isPresentableAsCurrent(s)).toBe(false); expect(NON_CURRENT_STATES).toContain('UNKNOWN'); }); const cases: Array<[string, Parameters[0], string]> = [ [ 'recent load', { lastSuccessfulLoadAt: '2026-09-07T11:50:00Z', lastRunStatus: 'succeeded', policy, now: NOW }, 'FRESH', ], [ 'behind schedule', { lastSuccessfulLoadAt: '2026-09-07T09:50:00Z', lastRunStatus: 'succeeded', policy, now: NOW }, 'DELAYED', ], [ 'well past the threshold', { lastSuccessfulLoadAt: '2026-09-06T12:00:00Z', lastRunStatus: 'succeeded', policy, now: NOW }, 'STALE', ], [ 'the last run FAILED, even though the data looks recent', { lastSuccessfulLoadAt: '2026-09-07T11:59:00Z', lastRunStatus: 'failed', policy, now: NOW }, 'FAILED', ], ['never run', { lastRunStatus: 'never_run', policy, now: NOW }, 'UNKNOWN'], ['no load timestamp at all', { policy, now: NOW }, 'UNKNOWN'], ['an unparseable timestamp', { lastSuccessfulLoadAt: 'not a date', policy, now: NOW }, 'UNKNOWN'], ]; it.each(cases)('%s -> %s', (_label, input, expected) => { expect(computeFreshness(input).state).toBe(expected); }); it('FAILED outranks a recent timestamp — a broken pipeline must not look FRESH', () => { const env = computeFreshness({ lastSuccessfulLoadAt: '2026-09-07T11:59:59Z', lastRunStatus: 'failed', policy, now: NOW, }); expect(env.state).toBe('FAILED'); expect(env.disclosure).toContain('FAILED'); }); it('every state carries a disclosure a UI can render verbatim', () => { for (const [, input] of cases) { const env = computeFreshness(input); expect(env.disclosure.length).toBeGreaterThan(10); } }); }); describe('BI-P026 DoD — stale data is never SILENTLY presented as current', () => { const env = computeFreshness({ lastSuccessfulLoadAt: '2026-09-01T00:00:00Z', lastRunStatus: 'succeeded', policy: { expectedIntervalMs: 3_600_000 }, now: Date.parse('2026-09-07T12:00:00Z'), }); it('refuses to release a result with NO freshness envelope', () => { // Not defaulted to FRESH, not defaulted to UNKNOWN: unreleasable. "We // forgot to attach it" and "the data is current" are indistinguishable to // a reader, so a default would make forgetting the silent path. expect(() => assertFreshnessDisclosed({}, 'dashboard panel p1')).toThrowError(SiaxError); }); it('refuses to release STALE data that the consumer has not acknowledged', () => { expect(env.state).toBe('STALE'); let thrown: unknown; try { assertFreshnessDisclosed({ freshness: env }, 'dashboard panel p1'); } catch (e) { thrown = e; } expect(thrown).toBeInstanceOf(SiaxError); expect((thrown as SiaxError).message).toContain('STALE'); }); it('releases STALE data once the consumer has acknowledged it', () => { expect(assertFreshnessDisclosed({ freshness: env, acknowledged_non_current: true }, 'p1').state).toBe( 'STALE', ); }); it.each(NON_CURRENT_STATES)('refuses an unacknowledged %s result', (state) => { expect(() => assertFreshnessDisclosed({ freshness: { ...env, state } }, 'p1')).toThrowError(SiaxError); }); it('releases FRESH data without ceremony', () => { expect(assertFreshnessDisclosed({ freshness: { ...env, state: 'FRESH' } }, 'p1').state).toBe('FRESH'); }); it('refuses an envelope carrying a state that is not one of the five', () => { expect(() => assertFreshnessDisclosed({ freshness: { ...env, state: 'PROBABLY_FINE' as never } }, 'p1'), ).toThrowError(SiaxError); }); });