0pt/tests/integration/semantic-parity.integration.test.ts

/**
 * tests/integration/semantic-parity.integration.test.ts
 * ======================================================
 * BI-P029 — the parity test `src/analytics/metrics/semantic-provider.ts` has
 * been citing.
 *
 * IT DID NOT EXIST. The provider's header named this path as the proof of the
 * one DoD surface it claimed was closed, and `providerReadiness().proven`
 * asserted that "0pt's metric compiler and an independent spec-driven renderer
 * return the same number for the golden metric against live ClickHouse data".
 * Nothing in the repository executed that comparison. A citation to a file
 * that is not there is the same failure as a green check that gates nothing:
 * it survives review because the sentence reads like evidence.
 *
 * So this file is the comparison, run for real:
 *
 *   PATH A  parseAnalyticsQueryRequest -> resolveAnalyticsQuery ->
 *           compileRequestToClickHouse -> live ClickHouse.
 *   PATH B  the same ResolvedQuery -> compileRequestToSemantic ->
 *           MetricFlowProvider.compile -> `renderIndependently` below, which
 *           reads ONLY the provider's structured query object and the registry
 *           model, writes its own SQL with its own quoting, and shares no line
 *           of code with compile-clickhouse.ts -> the same live ClickHouse.
 *
 * The two paths must return the same numbers. `renderIndependently` inlines
 * literals where compile-clickhouse binds parameters, groups by the physical
 * column expression rather than by a compiler alias, and orders in SQL rather
 * than by alias — so the SQL strings genuinely differ, and the test asserts
 * that they do. Two identical strings would make the parity claim vacuous.
 *
 * WHAT THIS STILL DOES NOT PROVE, and `providerReadiness()` still says so:
 * MetricFlow itself is not executed here (no Python runtime is provisioned for
 * this repo), and neither the Superset path nor the AI Analyst path is
 * compared. Three of BI-P029's four DoD surfaces remain open. This closes the
 * first one honestly rather than closing all four by assertion.
 *
 *   CLICKHOUSE_URL=http://127.0.0.1:28123 CLICKHOUSE_AUTH=default:test \
 *     pnpm vitest run tests/integration/semantic-parity.integration.test.ts
 *
 * It CREATEs and DROPs its own database. Never point it at the b0ts
 * ClickHouse or at any quarantined dataset.
 */
import fs from 'node:fs';
import path from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { ClickHouseAnalyticsEngine } from '@/src/analytics/engine/clickhouse-engine';
import {
  parseAnalyticsQueryRequest,
  queryResolutionFrom,
  resolveAnalyticsQuery,
  type ResolvedQuery,
} from '@/src/analytics/contract/query-request';
import { compileRequestToClickHouse } from '@/src/analytics/contract/compile-clickhouse';
import { compileRequestToSemantic } from '@/src/analytics/contract/compile-semantic';
import { MetricFlowProvider } from '@/src/analytics/metrics/semantic-provider';
import {
  InMemorySemanticModelStore,
  SemanticModelRegistry,
  type SemanticModelVersion,
} from '@/src/analytics/metrics/semantic-registry';
import type { MetricSpec } from '@/src/analytics/metrics/metric-model';


/**
 * Raw events carry a 30-day TTL (src/analytics/engine/schema.ts). Fixed
 * historical timestamps silently expire and make live-ClickHouse suites read
 * 0 rows, so seed rows are stamped relative to now (UTC, ClickHouse
 * DateTime64(3) text form).
 */
const SEED_EVENT_TIME = new Date(Date.now() - 2 * 3600_000).toISOString().replace("T", " ").replace("Z", "");

const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL;
const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH;
// Collection-time, never inside a hook: `describe.skipIf(flagSetInBeforeAll)`
// evaluates its flag before any hook has run and therefore skips ALWAYS.
const run = CLICKHOUSE_URL ? describe : describe.skip;

const DB = `opt_parity_${Date.now().toString(36)}`;
const TENANT = 't_alpha';
const PROJECT = 'p_main';

/**
 * The golden metric. It carries a DEFINITION FILTER (`channel != 'internal'`)
 * on purpose: a metric whose meaning is only "sum this column" cannot tell a
 * parity test anything, because every compiler agrees about a bare sum. The
 * business meaning of net revenue is that internal transfers are not revenue,
 * and that meaning has to survive both compile paths or the two paths are
 * answering different questions with the same metric id.
 */
const REVENUE: MetricSpec = {
  id: 'm_revenue',
  name: 'net_revenue',
  description: 'Invoiced revenue net of credit notes, in SEK, recognised on invoice date.',
  type: 'simple',
  measure: 'amount_sek',
  aggregation: 'sum',
  dimensions: ['country', 'channel'],
  filters: [{ dimension: 'channel', operator: 'ne', values: ['internal'] }],
  timeDimension: 'invoiced_at',
  unit: 'SEK',
  currency: 'SEK',
  format: 'currency',
  owner: 'finance@siax.io',
  version: 3,
  status: 'ACTIVE',
};

/** dimension name -> the key stored in `dim_names`. One mapping, both paths. */
const FIELD_OF: Record<string, string> = { country: 'country_code', channel: 'sales_channel' };

function semanticModel(version = 1): Omit<SemanticModelVersion, 'hash'> {
  return {
    model_id: 'sm_sales',
    version,
    tenant_id: TENANT,
    datasets: ['ds_sales'],
    relationships: [],
    metrics: [REVENUE],
    dimensions: [
      { id: 'd_country', name: 'country', dataset_id: 'ds_sales', kind: 'categorical' },
      { id: 'd_channel', name: 'channel', dataset_id: 'ds_sales', kind: 'categorical' },
      { id: 'd_invoiced', name: 'invoiced_at', dataset_id: 'ds_sales', kind: 'time' },
    ],
    valid_from: '2026-09-01T00:00:00Z',
    status: 'draft',
  };
}

/**
 * The fixture. Deliberately arranged so that dropping the metric's definition
 * filter changes the answer for EVERY country — a fixture where internal rows
 * happen to sum to zero would let a broken compiler pass.
 */
const ROWS: Array<{ country: string; channel: string; amount: number }> = [
  { country: 'SE', channel: 'direct', amount: 1000 },
  { country: 'SE', channel: 'partner', amount: 250 },
  { country: 'SE', channel: 'internal', amount: 9999 },
  { country: 'NO', channel: 'direct', amount: 400 },
  { country: 'NO', channel: 'internal', amount: 7777 },
  { country: 'DK', channel: 'partner', amount: 125.5 },
  { country: 'DK', channel: 'internal', amount: 3333 },
];

/** What the metric MEANS, computed in TypeScript, independent of any SQL. */
const EXPECTED: Record<string, number> = ROWS.filter((r) => r.channel !== 'internal').reduce(
  (acc, r) => ({ ...acc, [r.country]: (acc[r.country] ?? 0) + r.amount }),
  {} as Record<string, number>,
);

interface ChRow {
  [column: string]: string | number;
}

async function chQuery(sql: string, params: Record<string, string> = {}): Promise<ChRow[]> {
  const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid');
  for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
  const headers: Record<string, string> = {};
  if (CLICKHOUSE_AUTH) headers.Authorization = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}`;
  const res = await fetch(url.toString(), { method: 'POST', headers, body: sql });
  const text = await res.text();
  if (!res.ok) throw new Error(`ClickHouse ${res.status}: ${text.slice(0, 400)}`);
  if (!text.trim()) return [];
  return (JSON.parse(text) as { data: ChRow[] }).data;
}

async function chExec(sql: string, body?: string): Promise<void> {
  const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid');
  url.searchParams.set('query', sql);
  const headers: Record<string, string> = {};
  if (CLICKHOUSE_AUTH) headers.Authorization = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}`;
  const res = await fetch(url.toString(), { method: 'POST', headers, body: body ?? '' });
  if (!res.ok) throw new Error(`ClickHouse ${res.status}: ${(await res.text()).slice(0, 400)}`);
}

// ── PATH B: the independent renderer ───────────────────────────────────────

/** Single-quote escaping, written here because this renderer shares nothing. */
function lit(value: string): string {
  return `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
}

function predicate(field: string, operator: string, values: string[]): string {
  const lhs = `dim_values[indexOf(dim_names, ${lit(field)})]`;
  switch (operator) {
    case 'eq':
      return `${lhs} = ${lit(values[0])}`;
    case 'ne':
      return `${lhs} <> ${lit(values[0])}`;
    case 'in':
      return `${lhs} IN (${values.map(lit).join(',')})`;
    case 'not_in':
      return `${lhs} NOT IN (${values.map(lit).join(',')})`;
    default:
      throw new Error(`independent renderer does not implement operator ${operator}`);
  }
}

/**
 * Render the provider's structured query into SQL, reading ONLY that object
 * and the registry model. It never sees the ResolvedQuery, the ClickHouse
 * compiler's output, or its parameter map.
 *
 * The aggregate is deliberately expressed differently from compile-clickhouse:
 * a `sum(if(...))` rather than a `sumIf(...)`, grouped by the raw expression
 * rather than by an alias. Same arithmetic, different SQL — which is the only
 * way the comparison means anything.
 */
function renderIndependently(
  compiled: ReturnType<MetricFlowProvider['compile']>,
  model: SemanticModelVersion,
): string {
  const query = compiled.query as {
    metrics: string[];
    group_by: string[];
    where: Array<{ dimension: string; operator: string; values: string[] }>;
    limit: number;
    metric_filters?: Record<string, Array<{ dimension: string; operator: string; values: string[] }>>;
  };
  const byId = new Map(model.metrics.map((m) => [m.id, m]));

  const selects: string[] = [];
  const groups: string[] = [];
  for (const dimension of query.group_by) {
    const expr = `dim_values[indexOf(dim_names, ${lit(FIELD_OF[dimension])})]`;
    selects.push(`${expr} AS ${dimension}`);
    groups.push(expr);
  }
  for (const metricId of query.metrics) {
    const spec = byId.get(metricId);
    if (!spec) throw new Error(`metric ${metricId} is not in the model version`);
    // The metric's own definition filters. The provider query carries them
    // already resolved onto physical keys; the registry model states them in
    // dimension names. Either way they are the metric's MEANING, and a
    // renderer that ignored them would compute a different number.
    const own = query.metric_filters?.[metricId]
      ? query.metric_filters[metricId].map((f) => ({
          field: f.dimension,
          operator: f.operator,
          values: f.values,
        }))
      : spec.filters.map((f) => ({
          field: FIELD_OF[f.dimension],
          operator: f.operator,
          values: f.values ?? [],
        }));
    const conditions = [
      `metric = ${lit(spec.measure)}`,
      ...own.map((f) => predicate(f.field, f.operator, f.values)),
    ];
    selects.push(`sum(if(${conditions.join(' AND ')}, value, 0)) AS ${spec.id}`);
  }

  const where = [
    `tenant_id = ${lit(TENANT)}`,
    `project_id = ${lit(PROJECT)}`,
    `dataset_id = ${lit('ds_sales')}`,
    ...query.where.map((w) => predicate(FIELD_OF[w.dimension], w.operator, w.values ?? [])),
  ];

  return (
    `SELECT ${selects.join(', ')} FROM ${DB}.opt_analytics_event ` +
    `WHERE ${where.join(' AND ')} ` +
    (groups.length ? `GROUP BY ${groups.join(', ')} ORDER BY ${groups.join(', ')} ` : '') +
    `LIMIT ${query.limit} FORMAT JSON`
  );
}

// ── the suite ──────────────────────────────────────────────────────────────

let model: SemanticModelVersion;
let resolved: ResolvedQuery;
/** Filled by the regression test below; recorded in the evidence file. */
let preFix: Record<string, number> = {};

run('BI-P029 — one golden metric, two independent compilers, one live ClickHouse', () => {
  beforeAll(async () => {
    const engine = new ClickHouseAnalyticsEngine({
      baseUrl: CLICKHOUSE_URL!,
      basicAuth: CLICKHOUSE_AUTH,
      database: DB,
      dimensionCatalogs: { [TENANT]: FIELD_OF },
    });
    await engine.ensureSchema();

    const body = ROWS.map((r, i) =>
      JSON.stringify({
        tenant_id: TENANT,
        project_id: PROJECT,
        dataset_id: 'ds_sales',
        metric: 'amount_sek',
        event_id: `e${i}`,
        event_time: SEED_EVENT_TIME,
        value: r.amount,
        dim_names: ['country_code', 'sales_channel'],
        dim_values: [r.country, r.channel],
      }),
    ).join('\n');
    await chExec(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, body);

    model = await new SemanticModelRegistry(new InMemorySemanticModelStore()).publish(semanticModel(1));

    resolved = resolveAnalyticsQuery(
      parseAnalyticsQueryRequest({
        dataset: 'ds_sales',
        metrics: ['net_revenue'],
        dimensions: ['country'],
        orderBy: [{ ref: 'country', direction: 'asc' }],
      }),
      queryResolutionFrom({
        dataset_id: 'ds_sales',
        semantic_model_version: 1,
        metrics: [
          {
            name: 'net_revenue',
            metric_id: 'm_revenue',
            event_metric_key: 'amount_sek',
            aggregation: 'sum',
            allowed_dimensions: ['country', 'channel'],
            // The metric's OWN filters, resolved onto physical field names.
            definition_filters: REVENUE.filters.map((f) => ({
              field_name: FIELD_OF[f.dimension],
              operator: f.operator,
              values: f.values ?? [],
            })),
          },
        ],
        dimensions: [
          { name: 'country', dimension_id: 'd_country', field_name: 'country_code' },
          { name: 'channel', dimension_id: 'd_channel', field_name: 'sales_channel' },
        ],
      }),
    );
  }, 120_000);

  afterAll(async () => {
    if (CLICKHOUSE_URL) await chExec(`DROP DATABASE IF EXISTS ${DB}`);
  }, 60_000);

  it('the fixture is arranged so a dropped definition filter CHANGES every answer', () => {
    const withInternal = ROWS.reduce(
      (acc, r) => ({ ...acc, [r.country]: (acc[r.country] ?? 0) + r.amount }),
      {} as Record<string, number>,
    );
    for (const country of Object.keys(EXPECTED)) {
      expect(withInternal[country]).not.toBe(EXPECTED[country]);
    }
  });

  it('PATH A — 0pt’s ClickHouse compiler returns the metric’s meaning', async () => {
    const compiled = compileRequestToClickHouse(resolved, {
      tenant_id: TENANT,
      project_id: PROJECT,
      database: DB,
      maxResultRows: 1000,
    });
    const rows = await chQuery(compiled.sql, compiled.params);
    const got = Object.fromEntries(rows.map((r) => [String(r.dim_0), Number(r.metric_0)]));
    expect(got).toEqual(EXPECTED);
  });

  it('PATH B — an independent renderer driven only by the provider spec agrees', async () => {
    const spec = compileRequestToSemantic(resolved);
    const compiled = new MetricFlowProvider().compile(spec, model);
    expect(compiled.canonical_metric_ids).toEqual(['m_revenue']);

    const rows = await chQuery(renderIndependently(compiled, model));
    const got = Object.fromEntries(rows.map((r) => [String(r.country), Number(r.m_revenue)]));
    expect(got).toEqual(EXPECTED);
  });

  /**
   * The regression this suite was written for. Until BI-P029's gap review,
   * `compileRequestToClickHouse` had no notion of a metric's own filters:
   * `ResolvedMetric` did not carry them and the compiler never asked. The
   * metric definition said "excluding internal transfers" and the executed
   * SQL did not, so `net_revenue` returned a number that looked plausible and
   * was ~9x too large for this fixture.
   *
   * Re-run here by stripping the definition filters from the resolution, so
   * the number the bug produced is measured by the committed test rather than
   * remembered from a console.
   */
  it('a resolution WITHOUT the definition filters produces the pre-fix (wrong) number', async () => {
    const stripped: ResolvedQuery = {
      ...resolved,
      metrics: resolved.metrics.map((m) => ({ ...m, definition_filters: undefined })),
    };
    const compiled = compileRequestToClickHouse(stripped, {
      tenant_id: TENANT,
      project_id: PROJECT,
      database: DB,
      maxResultRows: 1000,
    });
    const rows = await chQuery(compiled.sql, compiled.params);
    preFix = Object.fromEntries(rows.map((r) => [String(r.dim_0), Number(r.metric_0)]));

    // Not merely different — OVERSTATED, on every country, by the internal
    // transfers the definition excludes.
    for (const country of Object.keys(EXPECTED)) {
      expect(preFix[country]).toBeGreaterThan(EXPECTED[country]);
    }
    expect(compiled.sql).not.toContain('mf0_0');
  });

  it('the golden metric returns the SAME number on both paths, within tolerance', async () => {
    const a = await chQuery(
      ...(() => {
        const c = compileRequestToClickHouse(resolved, {
          tenant_id: TENANT,
          project_id: PROJECT,
          database: DB,
          maxResultRows: 1000,
        });
        return [c.sql, c.params] as const;
      })(),
    );
    const b = await chQuery(
      renderIndependently(new MetricFlowProvider().compile(compileRequestToSemantic(resolved), model), model),
    );

    const left = Object.fromEntries(a.map((r) => [String(r.dim_0), Number(r.metric_0)]));
    const right = Object.fromEntries(b.map((r) => [String(r.country), Number(r.m_revenue)]));

    expect(Object.keys(left).sort()).toEqual(Object.keys(right).sort());
    for (const key of Object.keys(left)) {
      // Relative tolerance, because two different aggregate expressions over
      // Float64 may associate the additions differently.
      const denominator = Math.max(Math.abs(left[key]), 1);
      expect(Math.abs(left[key] - right[key]) / denominator).toBeLessThan(1e-9);
    }

    // Parity between two identical strings proves nothing.
    const sqlA = compileRequestToClickHouse(resolved, {
      tenant_id: TENANT,
      project_id: PROJECT,
      database: DB,
      maxResultRows: 1000,
    }).sql;
    const sqlB = renderIndependently(
      new MetricFlowProvider().compile(compileRequestToSemantic(resolved), model),
      model,
    );
    expect(sqlA).not.toBe(sqlB);
    expect(sqlA).toContain('sumIf(');
    expect(sqlB).toContain('sum(if(');

    fs.mkdirSync(path.join(process.cwd(), 'evidence', 'bi-p029'), { recursive: true });
    fs.writeFileSync(
      path.join(process.cwd(), 'evidence', 'bi-p029', 'semantic-parity.json'),
      `${JSON.stringify(
        {
          prompt: 'BI-P029',
          title: 'Golden metric parity: 0pt metric compiler vs an independent spec-driven renderer',
          recorded_at: new Date().toISOString(),
          clickhouse_version: (await chQuery('SELECT version() AS v FORMAT JSON'))[0]?.v,
          golden_metric: {
            id: REVENUE.id,
            name: REVENUE.name,
            definition_filters: REVENUE.filters,
            semantic_model: { model_id: model.model_id, version: model.version, hash: model.hash },
          },
          expected_from_typescript: EXPECTED,
          regression_guarded: {
            description:
              'the same request compiled WITHOUT the metric definition filters — what 0pt returned before this change',
            result: preFix,
          },
          path_a: { compiler: '0pt compileRequestToClickHouse', aggregate: 'sumIf', result: left },
          path_b: {
            compiler: 'MetricFlowProvider.compile + independent renderer',
            aggregate: 'sum(if(...))',
            result: right,
          },
          tolerance: '1e-9 relative',
          verdict: 'PASS',
          still_open: [
            'MetricFlow itself was not executed (no Python runtime provisioned for this repo)',
            'the Superset integration path is not implemented',
            'the AI Analyst path (inf0#29) has not been compared against this provider',
          ],
        },
        null,
        2,
      )}\n`,
    );
  });
});