diff --git a/src/analytics/dashboards/dashboard-document.ts b/src/analytics/dashboards/dashboard-document.ts new file mode 100644 index 0000000..788c8c6 --- /dev/null +++ b/src/analytics/dashboards/dashboard-document.ts @@ -0,0 +1,837 @@ +/** + * src/analytics/dashboards/dashboard-document.ts + * =============================================== + * BI-P060 — ANALYTICS DASHBOARD MODEL. + * + * BI-P002 already named `DashboardDefinition` and `DashboardVersion` and gave + * them storage (sql/008). What it did not give them is a BODY: `layout_json` + * is a string, and a string can hold anything — including a Superset export + * that only Superset can read back. This module is that body, and its whole + * point is the DoD: + * + * "A native SIAX dashboard can be reconstructed without Superset." + * + * Reconstruction is not a claim you can make by writing a type. It is provable + * exactly once you can take a document and produce, for every widget, a + * runnable `AnalyticsQueryRequest` (BI-P016) plus a rendering spec, with no + * vendor identifier anywhere in the plan. `planDashboard()` is that function + * and `tests/analytics/dashboard-model.test.ts` runs it over a document + * carrying Superset ids, asserting the plan is complete and vendor-free. + * + * THE VENDOR RULE, stated once: a Superset/Metabase/Cube identifier may be + * carried as `vendor_ref` for round-tripping, and it is + * - never a widget's source of truth (a widget with only a vendor_ref is + * REFUSED — it is precisely the document that cannot be reconstructed), + * - never part of the canonical hash (so re-exporting to a new Superset + * instance does not fabricate a new dashboard version). + * + * NO SQL LIVES HERE either. Widget sources are metric/dimension NAMES that + * resolve through the catalog, and query widgets are validated by + * `parseAnalyticsQueryRequest`, which rejects `sql`, `where`, `tenant_id` and + * every other smuggling key structurally. A dashboard is therefore not a new + * way into the engine; it is a saved set of contract requests. + */ +import crypto from 'node:crypto'; +import { SiaxError } from '../../platform/errors'; +import { + parseAnalyticsQueryRequest, + TIME_GRAINS, + type AnalyticsFilter, + type AnalyticsQueryRequest, + type FilterOperator, + type TimeGrain, +} from '../contract/query-request'; + +// ── vocabulary ───────────────────────────────────────────────────────────── + +/** + * Visualization types. A CLOSED list, deliberately engine-neutral: these are + * SIAX chart kinds (DES1GN BI-P051..P057 renders them), not Superset viz_type + * strings. A document naming `echarts_timeseries` is a Superset document. + */ +export const VISUALIZATION_TYPES = [ + 'big_number', + 'line', + 'area', + 'bar', + 'stacked_bar', + 'pie', + 'table', + 'pivot_table', + 'scatter', + 'heatmap', + 'funnel', + 'map', +] as const; +export type VisualizationType = (typeof VISUALIZATION_TYPES)[number]; + +export const REFRESH_MODES = ['manual', 'interval', 'scheduled'] as const; +export type RefreshMode = (typeof REFRESH_MODES)[number]; + +/** + * Who may open the dashboard. `roles` are cl0ud role names (BI-P033 decides, + * this only declares). There is deliberately NO tenant field: a dashboard is + * stored inside a tenant, and a document that could name one would be a way to + * ask for another tenant's dashboard — the INF0 Wave-2 shape. + */ +export const DASHBOARD_VISIBILITIES = ['private', 'tenant', 'roles'] as const; +export type DashboardVisibility = (typeof DASHBOARD_VISIBILITIES)[number]; + +export const PARAMETER_TYPES = ['string', 'number', 'date', 'enum'] as const; +export type ParameterType = (typeof PARAMETER_TYPES)[number]; + +// ── the document ─────────────────────────────────────────────────────────── + +/** Grid placement. A 12-column grid, matching the portal layout (BI-P062). */ +export interface WidgetLayout { + x: number; + y: number; + w: number; + h: number; +} + +/** A saved dashboard filter. Its VALUES are data; its dimension is a name. */ +export interface DashboardFilter { + filter_id: string; + label: string; + /** Caller-facing dimension name, resolved through the catalog downstream. */ + dimension: string; + operator: FilterOperator; + /** Default values, or `{{param}}` references to a declared parameter. */ + default_values?: string[]; + /** May a viewer change it? A locked filter is how row scoping is saved. */ + locked?: boolean; +} + +/** A declared parameter. Widgets and filters may reference it as `{{name}}`. */ +export interface DashboardParameter { + name: string; + type: ParameterType; + required: boolean; + default_value?: string; + /** Required for `enum`; a closed list is the only way a value can be trusted. */ + allowed_values?: string[]; +} + +/** A widget backed by ONE metric — the common case, and the cheapest to plan. */ +export interface MetricWidgetSource { + kind: 'metric'; + dataset: string; + metric: string; + dimensions?: string[]; + grain?: TimeGrain; + limit?: number; +} + +/** A widget backed by a saved contract request (still no SQL, by construction). */ +export interface QueryWidgetSource { + kind: 'query'; + /** Parsed with `parseAnalyticsQueryRequest` at validation time, not at render time. */ + request: AnalyticsQueryRequest; +} + +export type WidgetSource = MetricWidgetSource | QueryWidgetSource; + +export interface VisualizationSpec { + type: VisualizationType; + /** Field roles by NAME. `x`/`y` name metrics or dimensions in the source. */ + encoding: { + x?: string; + y?: string[]; + series?: string; + value?: string; + }; + /** Presentation-only knobs. Bounded scalars; never code, never a template. */ + options?: Record; +} + +export interface DashboardWidget { + widget_id: string; + title: string; + source: WidgetSource; + visualization: VisualizationSpec; + layout: WidgetLayout; + /** filter_ids this widget honours. Empty means "all dashboard filters". */ + filter_bindings?: string[]; + /** Advisory only: excluded from the canonical hash, never a source of truth. */ + vendor_ref?: VendorRef; +} + +/** A foreign system's id for the same object. Round-tripping aid, nothing more. */ +export interface VendorRef { + system: string; + id: string; +} + +export interface DashboardRefresh { + mode: RefreshMode; + /** Required for `interval`. */ + interval_seconds?: number; + /** Required for `scheduled`. */ + cron?: string; + /** + * The staleness the OWNER accepts. BI-P026 computes actual freshness; this + * is the number that makes a freshness verdict actionable rather than + * decorative, and BI-P105 reads it when deciding whether a pre-aggregate may + * serve this dashboard. + */ + max_staleness_seconds: number; +} + +export interface DashboardPermissions { + visibility: DashboardVisibility; + /** Required (non-empty) when visibility is `roles`. */ + roles?: string[]; + /** Optional cl0ud entitlement key (BI-P079). */ + required_entitlement?: string; +} + +/** BI-P060's canonical document, field-for-field with the program's list. */ +export interface DashboardDocument { + id: string; + title: string; + description?: string; + layout: { grid_columns: number; row_height_px: number }; + filters: DashboardFilter[]; + widgets: DashboardWidget[]; + parameters: DashboardParameter[]; + refresh: DashboardRefresh; + permissions: DashboardPermissions; + /** Document schema version — bumped when THIS shape changes, not per edit. */ + version: number; + vendor_ref?: VendorRef; +} + +export const DASHBOARD_DOCUMENT_KEYS = [ + 'id', + 'title', + 'description', + 'layout', + 'filters', + 'widgets', + 'parameters', + 'refresh', + 'permissions', + 'version', + 'vendor_ref', +] as const; + +/** + * Keys that mean "this document is really a Superset export". Named + * explicitly so the refusal explains the architecture instead of saying + * "unknown key"; the unknown-key rule below catches everything else anyway. + */ +const VENDOR_NATIVE_KEYS = new Set([ + 'slice_id', + 'slice_name', + 'viz_type', + 'datasource', + 'datasource_id', + 'datasource_type', + 'form_data', + 'query_context', + 'chart_id', + 'dashboard_id_superset', + 'position_json', + 'json_metadata', + 'card_id', + 'cube', +]); + +/** Keys that would turn a dashboard into an SQL surface. */ +const SQL_SMELLING_KEYS = new Set([ + 'sql', + 'query_text', + 'statement', + 'expression', + 'where', + 'having', + 'raw', + 'tenant', + 'tenant_id', + 'project_id', + 'database', + 'table', +]); + +const MAX_WIDGETS = 60; +const MAX_FILTERS = 32; +const MAX_PARAMETERS = 24; +const MAX_NAME_LENGTH = 128; +const MAX_TITLE_LENGTH = 200; +const MAX_GRID_COLUMNS = 24; +const MIN_REFRESH_INTERVAL_SECONDS = 10; +const MAX_STALENESS_SECONDS = 30 * 24 * 3600; +/** `{{param}}` — the only templating the document has, and it binds a VALUE. */ +const PARAM_REFERENCE = /^\{\{\s*([A-Za-z][A-Za-z0-9_]{0,63})\s*\}\}$/; +const IDENTIFIER = /^[A-Za-z][A-Za-z0-9_.-]{0,63}$/; + +function invalid(detail: string): never { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.dashboard', detail); +} + +function asObject(raw: unknown, what: string): Record { + if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) invalid(`${what} must be an object`); + return raw as Record; +} + +function rejectUnknownKeys(obj: Record, allowed: readonly string[], what: string): void { + for (const key of Object.keys(obj)) { + if (allowed.includes(key)) continue; + if (VENDOR_NATIVE_KEYS.has(key)) { + invalid( + `"${key}" in ${what} is a vendor-native field — a SIAX dashboard is defined by metrics, ` + + `queries and a visualization spec, not by a BI tool's internal ids (BI-P060). ` + + `Carry the id in "vendor_ref" if you need to round-trip it.`, + ); + } + if (SQL_SMELLING_KEYS.has(key)) { + invalid(`"${key}" in ${what} is not part of the dashboard model — dashboards do not carry SQL`); + } + invalid(`unknown field "${key}" in ${what}`); + } +} + +function asString(raw: unknown, what: string, max = MAX_NAME_LENGTH): string { + if (typeof raw !== 'string') invalid(`${what} must be a string`); + const v = raw.trim(); + if (!v) invalid(`${what} must not be empty`); + if (v.length > max) invalid(`${what} exceeds ${max} characters`); + return v; +} + +function asIdentifier(raw: unknown, what: string): string { + const v = asString(raw, what); + if (!IDENTIFIER.test(v)) invalid(`${what} is not a valid identifier`); + return v; +} + +function asInteger(raw: unknown, what: string, min: number, max: number): number { + if (typeof raw !== 'number' || !Number.isFinite(raw)) invalid(`${what} must be a finite number`); + if (!Number.isInteger(raw)) invalid(`${what} must be an integer`); + if (raw < min || raw > max) invalid(`${what} must be between ${min} and ${max}`); + return raw; +} + +function parseVendorRef(raw: unknown, what: string): VendorRef { + const obj = asObject(raw, what); + rejectUnknownKeys(obj, ['system', 'id'], what); + return { system: asString(obj.system, `${what}.system`), id: asString(obj.id, `${what}.id`, 256) }; +} + +// ── parsing ──────────────────────────────────────────────────────────────── + +function parseParameter(raw: unknown, i: number): DashboardParameter { + const what = `parameters[${i}]`; + const obj = asObject(raw, what); + rejectUnknownKeys(obj, ['name', 'type', 'required', 'default_value', 'allowed_values'], what); + const name = asIdentifier(obj.name, `${what}.name`); + const type = asString(obj.type, `${what}.type`); + if (!(PARAMETER_TYPES as readonly string[]).includes(type)) invalid(`${what}.type is not a parameter type`); + if (typeof obj.required !== 'boolean') invalid(`${what}.required must be a boolean`); + const p: DashboardParameter = { name, type: type as ParameterType, required: obj.required }; + if (obj.default_value !== undefined) + p.default_value = asString(obj.default_value, `${what}.default_value`, 4096); + if (obj.allowed_values !== undefined) { + if (!Array.isArray(obj.allowed_values) || obj.allowed_values.length === 0) + invalid(`${what}.allowed_values must be a non-empty array`); + p.allowed_values = obj.allowed_values.map((v, j) => asString(v, `${what}.allowed_values[${j}]`, 4096)); + } + if (p.type === 'enum' && !p.allowed_values) { + invalid(`${what} is an enum parameter with no allowed_values — an unbounded enum is a free-text field`); + } + if (p.required && p.default_value !== undefined) { + invalid(`${what} is required AND has a default — a default makes "required" unenforceable`); + } + return p; +} + +function parseFilter(raw: unknown, i: number, parameterNames: Set): DashboardFilter { + const what = `filters[${i}]`; + const obj = asObject(raw, what); + rejectUnknownKeys(obj, ['filter_id', 'label', 'dimension', 'operator', 'default_values', 'locked'], what); + const filter_id = asIdentifier(obj.filter_id, `${what}.filter_id`); + const label = asString(obj.label, `${what}.label`, MAX_TITLE_LENGTH); + const dimension = asString(obj.dimension, `${what}.dimension`); + const operator = asString(obj.operator, `${what}.operator`) as FilterOperator; + const f: DashboardFilter = { filter_id, label, dimension, operator }; + if (obj.default_values !== undefined) { + if (!Array.isArray(obj.default_values)) invalid(`${what}.default_values must be an array`); + f.default_values = obj.default_values.map((v, j) => { + const s = asString(v, `${what}.default_values[${j}]`, 4096); + const m = PARAM_REFERENCE.exec(s); + if (m && !parameterNames.has(m[1])) { + invalid(`${what}.default_values[${j}] references undeclared parameter "${m[1]}"`); + } + return s; + }); + } + if (obj.locked !== undefined) { + if (typeof obj.locked !== 'boolean') invalid(`${what}.locked must be a boolean`); + f.locked = obj.locked; + } + return f; +} + +function parseVisualization(raw: unknown, what: string): VisualizationSpec { + const obj = asObject(raw, what); + rejectUnknownKeys(obj, ['type', 'encoding', 'options'], what); + const type = asString(obj.type, `${what}.type`); + if (!(VISUALIZATION_TYPES as readonly string[]).includes(type)) { + invalid( + `${what}.type "${type}" is not a SIAX visualization type — the canonical list is engine-neutral ` + + `so a chart survives a BI engine swap (BI-P060)`, + ); + } + const encodingRaw = asObject(obj.encoding ?? {}, `${what}.encoding`); + rejectUnknownKeys(encodingRaw, ['x', 'y', 'series', 'value'], `${what}.encoding`); + const encoding: VisualizationSpec['encoding'] = {}; + if (encodingRaw.x !== undefined) encoding.x = asString(encodingRaw.x, `${what}.encoding.x`); + if (encodingRaw.series !== undefined) + encoding.series = asString(encodingRaw.series, `${what}.encoding.series`); + if (encodingRaw.value !== undefined) encoding.value = asString(encodingRaw.value, `${what}.encoding.value`); + if (encodingRaw.y !== undefined) { + if (!Array.isArray(encodingRaw.y)) invalid(`${what}.encoding.y must be an array`); + encoding.y = encodingRaw.y.map((v, j) => asString(v, `${what}.encoding.y[${j}]`)); + } + const spec: VisualizationSpec = { type: type as VisualizationType, encoding }; + if (obj.options !== undefined) { + const options = asObject(obj.options, `${what}.options`); + const out: Record = {}; + for (const [k, v] of Object.entries(options)) { + if (!IDENTIFIER.test(k)) invalid(`${what}.options key "${k}" is not an identifier`); + if (typeof v === 'string') { + if (v.length > 256) invalid(`${what}.options.${k} exceeds 256 characters`); + out[k] = v; + } else if (typeof v === 'number') { + if (!Number.isFinite(v)) invalid(`${what}.options.${k} must be finite`); + out[k] = v; + } else if (typeof v === 'boolean') { + out[k] = v; + } else { + invalid(`${what}.options.${k} must be a string, number or boolean — options are not a payload`); + } + } + spec.options = out; + } + return spec; +} + +function parseSource(raw: unknown, what: string): WidgetSource { + const obj = asObject(raw, what); + const kind = asString(obj.kind, `${what}.kind`); + if (kind === 'metric') { + rejectUnknownKeys(obj, ['kind', 'dataset', 'metric', 'dimensions', 'grain', 'limit'], what); + const source: MetricWidgetSource = { + kind: 'metric', + dataset: asString(obj.dataset, `${what}.dataset`), + metric: asString(obj.metric, `${what}.metric`), + }; + if (obj.dimensions !== undefined) { + if (!Array.isArray(obj.dimensions)) invalid(`${what}.dimensions must be an array`); + source.dimensions = obj.dimensions.map((d, j) => asString(d, `${what}.dimensions[${j}]`)); + } + if (obj.grain !== undefined) { + const grain = asString(obj.grain, `${what}.grain`); + if (!(TIME_GRAINS as readonly string[]).includes(grain)) invalid(`${what}.grain is not a time grain`); + source.grain = grain as TimeGrain; + } + if (obj.limit !== undefined) source.limit = asInteger(obj.limit, `${what}.limit`, 1, 100_000); + return source; + } + if (kind === 'query') { + rejectUnknownKeys(obj, ['kind', 'request'], what); + // parseAnalyticsQueryRequest is the SQL boundary: it is why a query widget + // cannot become an SQL widget by adding a field. + return { kind: 'query', request: parseAnalyticsQueryRequest(obj.request) }; + } + invalid( + `${what}.kind must be "metric" or "query" — a widget with no reconstructable source cannot be ` + + `rendered without the tool that exported it (BI-P060 DoD)`, + ); +} + +function parseWidget(raw: unknown, i: number, gridColumns: number): DashboardWidget { + const what = `widgets[${i}]`; + const obj = asObject(raw, what); + rejectUnknownKeys( + obj, + ['widget_id', 'title', 'source', 'visualization', 'layout', 'filter_bindings', 'vendor_ref'], + what, + ); + const widget_id = asIdentifier(obj.widget_id, `${what}.widget_id`); + const title = asString(obj.title, `${what}.title`, MAX_TITLE_LENGTH); + const source = parseSource(obj.source, `${what}.source`); + const visualization = parseVisualization(obj.visualization, `${what}.visualization`); + + const layoutObj = asObject(obj.layout, `${what}.layout`); + rejectUnknownKeys(layoutObj, ['x', 'y', 'w', 'h'], `${what}.layout`); + const layout: WidgetLayout = { + x: asInteger(layoutObj.x, `${what}.layout.x`, 0, gridColumns - 1), + y: asInteger(layoutObj.y, `${what}.layout.y`, 0, 10_000), + w: asInteger(layoutObj.w, `${what}.layout.w`, 1, gridColumns), + h: asInteger(layoutObj.h, `${what}.layout.h`, 1, 200), + }; + if (layout.x + layout.w > gridColumns) { + invalid(`${what}.layout overflows the ${gridColumns}-column grid (x=${layout.x}, w=${layout.w})`); + } + + const widget: DashboardWidget = { widget_id, title, source, visualization, layout }; + if (obj.filter_bindings !== undefined) { + if (!Array.isArray(obj.filter_bindings)) invalid(`${what}.filter_bindings must be an array`); + widget.filter_bindings = obj.filter_bindings.map((b, j) => + asIdentifier(b, `${what}.filter_bindings[${j}]`), + ); + } + if (obj.vendor_ref !== undefined) widget.vendor_ref = parseVendorRef(obj.vendor_ref, `${what}.vendor_ref`); + return widget; +} + +/** + * Parse an untrusted dashboard document. Fail-closed everywhere: a field this + * function does not understand is a refusal, because "ignored for now" is how + * a Superset-only document ends up stored as canonical truth. + */ +export function parseDashboardDocument(raw: unknown): DashboardDocument { + const obj = asObject(raw, 'dashboard document'); + rejectUnknownKeys(obj, DASHBOARD_DOCUMENT_KEYS, 'dashboard document'); + + const id = asIdentifier(obj.id, 'id'); + const title = asString(obj.title, 'title', MAX_TITLE_LENGTH); + const version = asInteger(obj.version, 'version', 1, 1_000_000); + + const layoutObj = asObject(obj.layout, 'layout'); + rejectUnknownKeys(layoutObj, ['grid_columns', 'row_height_px'], 'layout'); + const layout = { + grid_columns: asInteger(layoutObj.grid_columns, 'layout.grid_columns', 1, MAX_GRID_COLUMNS), + row_height_px: asInteger(layoutObj.row_height_px, 'layout.row_height_px', 8, 400), + }; + + const rawParameters = obj.parameters ?? []; + if (!Array.isArray(rawParameters)) invalid('parameters must be an array'); + if (rawParameters.length > MAX_PARAMETERS) invalid(`at most ${MAX_PARAMETERS} parameters`); + const parameters = rawParameters.map((p, i) => parseParameter(p, i)); + const parameterNames = new Set(parameters.map((p) => p.name)); + if (parameterNames.size !== parameters.length) invalid('parameters contains duplicate names'); + + const rawFilters = obj.filters ?? []; + if (!Array.isArray(rawFilters)) invalid('filters must be an array'); + if (rawFilters.length > MAX_FILTERS) invalid(`at most ${MAX_FILTERS} filters`); + const filters = rawFilters.map((f, i) => parseFilter(f, i, parameterNames)); + const filterIds = new Set(filters.map((f) => f.filter_id)); + if (filterIds.size !== filters.length) invalid('filters contains duplicate filter_id'); + + if (!Array.isArray(obj.widgets)) invalid('widgets must be an array'); + if (obj.widgets.length === 0) invalid('a dashboard with no widgets has nothing to reconstruct'); + if (obj.widgets.length > MAX_WIDGETS) invalid(`at most ${MAX_WIDGETS} widgets`); + const widgets = obj.widgets.map((w, i) => parseWidget(w, i, layout.grid_columns)); + if (new Set(widgets.map((w) => w.widget_id)).size !== widgets.length) { + invalid('widgets contains duplicate widget_id'); + } + for (const w of widgets) { + for (const b of w.filter_bindings ?? []) { + if (!filterIds.has(b)) invalid(`widget "${w.widget_id}" binds undeclared filter "${b}"`); + } + } + + const refreshObj = asObject(obj.refresh, 'refresh'); + rejectUnknownKeys(refreshObj, ['mode', 'interval_seconds', 'cron', 'max_staleness_seconds'], 'refresh'); + const mode = asString(refreshObj.mode, 'refresh.mode'); + if (!(REFRESH_MODES as readonly string[]).includes(mode)) invalid('refresh.mode is not a refresh mode'); + const refresh: DashboardRefresh = { + mode: mode as RefreshMode, + max_staleness_seconds: asInteger( + refreshObj.max_staleness_seconds, + 'refresh.max_staleness_seconds', + 1, + MAX_STALENESS_SECONDS, + ), + }; + if (refresh.mode === 'interval') { + refresh.interval_seconds = asInteger( + refreshObj.interval_seconds, + 'refresh.interval_seconds', + MIN_REFRESH_INTERVAL_SECONDS, + 86_400, + ); + if (refreshObj.cron !== undefined) invalid('refresh.cron is meaningless when mode is "interval"'); + } else if (refresh.mode === 'scheduled') { + refresh.cron = asString(refreshObj.cron, 'refresh.cron', 128); + if (refreshObj.interval_seconds !== undefined) { + invalid('refresh.interval_seconds is meaningless when mode is "scheduled"'); + } + } else if (refreshObj.interval_seconds !== undefined || refreshObj.cron !== undefined) { + invalid('refresh.mode is "manual" but a schedule was supplied'); + } + + const permObj = asObject(obj.permissions, 'permissions'); + rejectUnknownKeys(permObj, ['visibility', 'roles', 'required_entitlement'], 'permissions'); + const visibility = asString(permObj.visibility, 'permissions.visibility'); + if (!(DASHBOARD_VISIBILITIES as readonly string[]).includes(visibility)) { + invalid('permissions.visibility is not a visibility'); + } + const permissions: DashboardPermissions = { visibility: visibility as DashboardVisibility }; + if (permObj.roles !== undefined) { + if (!Array.isArray(permObj.roles)) invalid('permissions.roles must be an array'); + permissions.roles = permObj.roles.map((r, i) => asIdentifier(r, `permissions.roles[${i}]`)); + } + if (permissions.visibility === 'roles' && (permissions.roles ?? []).length === 0) { + invalid( + 'permissions.visibility is "roles" with no roles — an empty role list reads as "deny all" to one ' + + 'implementation and "allow all" to the next; say which', + ); + } + if (permObj.required_entitlement !== undefined) { + permissions.required_entitlement = asIdentifier( + permObj.required_entitlement, + 'permissions.required_entitlement', + ); + } + + const doc: DashboardDocument = { + id, + title, + layout, + filters, + widgets, + parameters, + refresh, + permissions, + version, + }; + if (obj.description !== undefined) doc.description = asString(obj.description, 'description', 2000); + if (obj.vendor_ref !== undefined) doc.vendor_ref = parseVendorRef(obj.vendor_ref, 'vendor_ref'); + return doc; +} + +// ── canonical hash ───────────────────────────────────────────────────────── + +/** + * Stable ordering of every field that MEANS something, with `vendor_ref` + * omitted at both levels. Two documents that render identically hash + * identically even if their JSON key order differs or one of them has been + * round-tripped through a BI tool; a document whose meaning changed hashes + * differently. That is what makes `DashboardVersion.layout_hash` a usable + * identity for BI-P061 rollback. + */ +function canonicalize(doc: DashboardDocument): unknown { + return [ + doc.id, + doc.title, + doc.description ?? '', + doc.version, + [doc.layout.grid_columns, doc.layout.row_height_px], + doc.parameters + .map((p) => [p.name, p.type, p.required, p.default_value ?? '', [...(p.allowed_values ?? [])].sort()]) + .sort((a, b) => String(a[0]).localeCompare(String(b[0]))), + doc.filters + .map((f) => [f.filter_id, f.label, f.dimension, f.operator, f.default_values ?? [], f.locked ?? false]) + .sort((a, b) => String(a[0]).localeCompare(String(b[0]))), + doc.widgets + .map((w) => [ + w.widget_id, + w.title, + w.source.kind === 'metric' + ? [ + 'metric', + w.source.dataset, + w.source.metric, + w.source.dimensions ?? [], + w.source.grain ?? '', + w.source.limit ?? 0, + ] + : ['query', canonicalRequest(w.source.request)], + [ + w.visualization.type, + w.visualization.encoding.x ?? '', + w.visualization.encoding.y ?? [], + w.visualization.encoding.series ?? '', + w.visualization.encoding.value ?? '', + Object.keys(w.visualization.options ?? {}) + .sort() + .map((k) => [k, (w.visualization.options ?? {})[k]]), + ], + [w.layout.x, w.layout.y, w.layout.w, w.layout.h], + [...(w.filter_bindings ?? [])].sort(), + ]) + .sort((a, b) => String(a[0]).localeCompare(String(b[0]))), + [ + doc.refresh.mode, + doc.refresh.interval_seconds ?? 0, + doc.refresh.cron ?? '', + doc.refresh.max_staleness_seconds, + ], + [ + doc.permissions.visibility, + [...(doc.permissions.roles ?? [])].sort(), + doc.permissions.required_entitlement ?? '', + ], + ]; +} + +function canonicalRequest(r: AnalyticsQueryRequest): unknown { + return [ + r.dataset, + [...r.metrics], + [...r.dimensions], + r.filters.map((f) => [f.dimension, f.operator, f.values ?? []]), + r.timeRange + ? [r.timeRange.dimension ?? '', r.timeRange.from ?? '', r.timeRange.to ?? '', r.timeRange.grain ?? ''] + : [], + (r.orderBy ?? []).map((o) => [o.ref, o.direction]), + r.limit ?? 0, + r.format ?? 'json', + ]; +} + +/** sha256 of the canonical form. Fills `DashboardVersion.layout_hash`. */ +export function dashboardDocumentHash(doc: DashboardDocument): string { + return crypto + .createHash('sha256') + .update(JSON.stringify(canonicalize(doc))) + .digest('hex'); +} + +// ── reconstruction (the DoD) ─────────────────────────────────────────────── + +export interface PlannedWidget { + widget_id: string; + title: string; + /** Runnable, contract-valid, engine-neutral. This is the DoD in one field. */ + request: AnalyticsQueryRequest; + visualization: VisualizationSpec; + layout: WidgetLayout; +} + +export interface DashboardPlan { + dashboard_id: string; + title: string; + document_hash: string; + grid_columns: number; + widgets: PlannedWidget[]; + /** Parameter values actually bound, after defaults. Reported, not implied. */ + bound_parameters: Record; +} + +export interface PlanOptions { + /** Caller-supplied parameter values. Validated against the declarations. */ + parameters?: Record; + /** Filter ids a viewer switched off. `locked` filters cannot be switched off. */ + disabled_filter_ids?: string[]; +} + +function bindParameters(doc: DashboardDocument, supplied: Record): Record { + const bound: Record = {}; + const declared = new Set(doc.parameters.map((p) => p.name)); + for (const name of Object.keys(supplied)) { + if (!declared.has(name)) invalid(`parameter "${name}" is not declared by this dashboard`); + } + for (const p of doc.parameters) { + const value = supplied[p.name] ?? p.default_value; + if (value === undefined) { + if (p.required) invalid(`required parameter "${p.name}" was not supplied`); + continue; + } + if (p.allowed_values && !p.allowed_values.includes(value)) { + invalid(`parameter "${p.name}" value is not in its allowed_values`); + } + if (p.type === 'number' && !Number.isFinite(Number(value))) { + invalid(`parameter "${p.name}" is declared number but "${value}" is not numeric`); + } + if (p.type === 'date' && Number.isNaN(Date.parse(value))) { + invalid(`parameter "${p.name}" is declared date but "${value}" is not an ISO date`); + } + bound[p.name] = value; + } + return bound; +} + +function resolveFilterValues(f: DashboardFilter, bound: Record): string[] | undefined { + if (!f.default_values) return undefined; + return f.default_values.map((v) => { + const m = PARAM_REFERENCE.exec(v); + if (!m) return v; + const value = bound[m[1]]; + if (value === undefined) { + invalid(`filter "${f.filter_id}" references parameter "${m[1]}", which has no value`); + } + return value; + }); +} + +/** + * Turn a dashboard document into runnable analytics requests. + * + * This IS BI-P060's DoD. If this function returns a plan, the dashboard can be + * rendered by anything that can execute `AnalyticsQueryRequest` and draw the + * twelve visualization types — Superset not required, not consulted, and not + * present anywhere in the output. + */ +export function planDashboard(doc: DashboardDocument, options: PlanOptions = {}): DashboardPlan { + const bound = bindParameters(doc, options.parameters ?? {}); + const disabled = new Set(options.disabled_filter_ids ?? []); + for (const f of doc.filters) { + if (disabled.has(f.filter_id) && f.locked) { + invalid( + `filter "${f.filter_id}" is locked and cannot be disabled — locked filters are how a saved ` + + `dashboard pins its scope, and a viewer removing one silently widens the result set`, + ); + } + } + + const widgets = doc.widgets.map((w) => { + const applicable = doc.filters.filter( + (f) => + !disabled.has(f.filter_id) && + (w.filter_bindings === undefined || w.filter_bindings.length === 0 + ? true + : w.filter_bindings.includes(f.filter_id)), + ); + const dashboardFilters: AnalyticsFilter[] = applicable.map((f) => { + const values = resolveFilterValues(f, bound); + const af: AnalyticsFilter = { dimension: f.dimension, operator: f.operator }; + if (values !== undefined) af.values = values; + return af; + }); + + const base = + w.source.kind === 'metric' + ? { + dataset: w.source.dataset, + metrics: [w.source.metric], + dimensions: w.source.dimensions ?? [], + filters: [] as AnalyticsFilter[], + ...(w.source.grain ? { timeRange: { grain: w.source.grain } } : {}), + ...(w.source.limit ? { limit: w.source.limit } : {}), + } + : { + ...w.source.request, + timeRange: w.source.request.timeRange, + }; + + // Re-parse the merged request rather than trusting the merge: the filters + // we just added come from the document, and a document is untrusted input + // even after it has been stored. + const request = parseAnalyticsQueryRequest({ + ...base, + filters: [...(w.source.kind === 'query' ? w.source.request.filters : []), ...dashboardFilters], + }); + + return { + widget_id: w.widget_id, + title: w.title, + request, + visualization: w.visualization, + layout: w.layout, + }; + }); + + return { + dashboard_id: doc.id, + title: doc.title, + document_hash: dashboardDocumentHash(doc), + grid_columns: doc.layout.grid_columns, + widgets, + bound_parameters: bound, + }; +} diff --git a/tests/analytics/dashboard-model.test.ts b/tests/analytics/dashboard-model.test.ts new file mode 100644 index 0000000..7c93bcf --- /dev/null +++ b/tests/analytics/dashboard-model.test.ts @@ -0,0 +1,330 @@ +/** + * tests/analytics/dashboard-model.test.ts + * ======================================== + * BI-P060 — ANALYTICS DASHBOARD MODEL. + * + * The DoD is "a native SIAX dashboard can be reconstructed without Superset", + * so the central test takes a document that CAME from Superset (it still + * carries the slice ids in `vendor_ref`), plans it, and then greps the whole + * plan for any trace of the vendor. A plan that still needs Superset to be + * understood fails here. + * + * The counterfactual matters as much: `vendor-only-widget-is-refused` builds + * the document that CANNOT be reconstructed — a widget whose only content is a + * Superset id — and requires the model to refuse it. Without that test the + * vendor-free assertion would pass trivially for documents nobody could + * render. + */ +import { describe, expect, it } from 'vitest'; +import { SiaxError } from '@/src/platform/errors'; +import { + dashboardDocumentHash, + parseDashboardDocument, + planDashboard, + VISUALIZATION_TYPES, + type DashboardDocument, +} from '@/src/analytics/dashboards/dashboard-document'; + +/** A dashboard exported from Superset and rewritten into the canonical model. */ +function supersetImportedDocument(): unknown { + return { + id: 'dash_sales_overview', + title: 'Sales overview', + description: 'Migrated from Superset dashboard 41.', + version: 1, + layout: { grid_columns: 12, row_height_px: 40 }, + parameters: [ + { + name: 'region', + type: 'enum', + required: false, + default_value: 'SE', + allowed_values: ['SE', 'NO', 'DK'], + }, + { name: 'since', type: 'date', required: true }, + ], + filters: [ + { + filter_id: 'f_region', + label: 'Region', + dimension: 'region', + operator: 'eq', + default_values: ['{{region}}'], + }, + { + filter_id: 'f_channel', + label: 'Channel', + dimension: 'channel', + operator: 'in', + default_values: ['web'], + }, + { + filter_id: 'f_tenant_scope', + label: 'Business unit', + dimension: 'business_unit', + operator: 'eq', + default_values: ['bu_north'], + locked: true, + }, + ], + widgets: [ + { + widget_id: 'w_revenue_total', + title: 'Revenue', + source: { kind: 'metric', dataset: 'ds_sales', metric: 'revenue' }, + visualization: { type: 'big_number', encoding: { value: 'revenue' } }, + layout: { x: 0, y: 0, w: 3, h: 2 }, + vendor_ref: { system: 'superset', id: 'slice-8812' }, + }, + { + widget_id: 'w_revenue_by_day', + title: 'Revenue by day', + source: { + kind: 'metric', + dataset: 'ds_sales', + metric: 'revenue', + dimensions: ['order_date'], + grain: 'day', + }, + visualization: { + type: 'line', + encoding: { x: 'order_date', y: ['revenue'] }, + options: { stacked: false }, + }, + layout: { x: 3, y: 0, w: 9, h: 4 }, + filter_bindings: ['f_region', 'f_tenant_scope'], + vendor_ref: { system: 'superset', id: 'slice-8813' }, + }, + { + widget_id: 'w_top_products', + title: 'Top products', + source: { + kind: 'query', + request: { + dataset: 'ds_sales', + metrics: ['revenue', 'orders'], + dimensions: ['product'], + filters: [{ dimension: 'status', operator: 'eq', values: ['completed'] }], + orderBy: [{ ref: 'revenue', direction: 'desc' }], + limit: 20, + }, + }, + visualization: { type: 'table', encoding: { x: 'product', y: ['revenue', 'orders'] } }, + layout: { x: 0, y: 4, w: 12, h: 6 }, + }, + ], + refresh: { mode: 'interval', interval_seconds: 300, max_staleness_seconds: 900 }, + permissions: { + visibility: 'roles', + roles: ['analyst', 'sales_lead'], + required_entitlement: 'bi_dashboards', + }, + vendor_ref: { system: 'superset', id: 'dashboard-41' }, + }; +} + +describe('BI-P060 dashboard model — reconstruction without Superset', () => { + it('plans every widget into a runnable analytics request', () => { + const doc = parseDashboardDocument(supersetImportedDocument()); + const plan = planDashboard(doc, { parameters: { since: '2026-01-01', region: 'NO' } }); + + expect(plan.widgets).toHaveLength(3); + for (const w of plan.widgets) { + expect(w.request.dataset).toBe('ds_sales'); + expect(w.request.metrics.length).toBeGreaterThan(0); + expect(VISUALIZATION_TYPES).toContain(w.visualization.type); + } + // The parameter reference was BOUND, not left as a template string. + const byDay = plan.widgets.find((w) => w.widget_id === 'w_revenue_by_day'); + expect(byDay?.request.filters.find((f) => f.dimension === 'region')?.values).toEqual(['NO']); + expect(plan.bound_parameters).toEqual({ region: 'NO', since: '2026-01-01' }); + }); + + it('the plan contains no vendor identifier anywhere', () => { + const doc = parseDashboardDocument(supersetImportedDocument()); + const plan = planDashboard(doc, { parameters: { since: '2026-01-01' } }); + const serialized = JSON.stringify(plan).toLowerCase(); + + // The detector must be capable of finding one: the document DOES carry + // these ids, so a grep that finds nothing in the plan means the plan is + // clean, not that the grep is broken. + expect(JSON.stringify(doc).toLowerCase()).toContain('slice-8812'); + + for (const trace of ['superset', 'slice-8812', 'slice-8813', 'dashboard-41', 'viz_type', 'form_data']) { + expect(serialized).not.toContain(trace); + } + }); + + it('vendor-only widget is refused — it is the document that cannot be reconstructed', () => { + const raw = supersetImportedDocument() as { widgets: unknown[] }; + raw.widgets = [ + { + widget_id: 'w_mystery', + title: 'Whatever Superset had', + source: { kind: 'vendor', system: 'superset', id: 'slice-9001' }, + visualization: { type: 'line', encoding: {} }, + layout: { x: 0, y: 0, w: 4, h: 3 }, + }, + ]; + expect(() => parseDashboardDocument(raw)).toThrow(SiaxError); + expect(() => parseDashboardDocument(raw)).toThrow(/reconstruct/i); + }); + + it('vendor-native fields are refused at the top level and inside a widget', () => { + const top = { ...(supersetImportedDocument() as object), position_json: '{}' }; + expect(() => parseDashboardDocument(top)).toThrow(/vendor-native/); + + const raw = supersetImportedDocument() as { widgets: Record[] }; + raw.widgets[0].viz_type = 'echarts_timeseries'; + expect(() => parseDashboardDocument(raw)).toThrow(/vendor-native/); + }); + + it('a Superset viz_type value is not a SIAX visualization type', () => { + const raw = supersetImportedDocument() as { widgets: Record[] }; + raw.widgets[0].visualization = { type: 'echarts_timeseries', encoding: {} }; + expect(() => parseDashboardDocument(raw)).toThrow(/not a SIAX visualization type/); + }); +}); + +describe('BI-P060 dashboard model — no SQL surface', () => { + it('a widget cannot carry SQL, however it is spelled', () => { + for (const key of ['sql', 'query_text', 'statement', 'where', 'expression']) { + const raw = supersetImportedDocument() as { widgets: Record[] }; + raw.widgets[0][key] = 'SELECT 1'; + expect(() => parseDashboardDocument(raw), key).toThrow(SiaxError); + } + }); + + it('a query widget goes through the BI-P016 contract, so tenant selection is impossible', () => { + const raw = supersetImportedDocument() as { widgets: Record[] }; + raw.widgets[2].source = { + kind: 'query', + request: { + dataset: 'ds_sales', + metrics: ['revenue'], + dimensions: [], + filters: [], + tenant_id: 't_victim', + }, + }; + expect(() => parseDashboardDocument(raw)).toThrow(/does not accept SQL, catalogs, or tenant selection/); + }); + + it('the document itself cannot name a tenant', () => { + const raw = { ...(supersetImportedDocument() as object), tenant_id: 't_victim' }; + expect(() => parseDashboardDocument(raw)).toThrow(/dashboards do not carry SQL|unknown field/); + }); +}); + +describe('BI-P060 dashboard model — validation that would otherwise bite at render time', () => { + it('a widget binding an undeclared filter is refused', () => { + const raw = supersetImportedDocument() as { widgets: Record[] }; + raw.widgets[1].filter_bindings = ['f_nonexistent']; + expect(() => parseDashboardDocument(raw)).toThrow(/undeclared filter/); + }); + + it('a filter defaulting to an undeclared parameter is refused', () => { + const raw = supersetImportedDocument() as { filters: Record[] }; + raw.filters[0].default_values = ['{{not_declared}}']; + expect(() => parseDashboardDocument(raw)).toThrow(/undeclared parameter/); + }); + + it('a widget overflowing the grid is refused', () => { + const raw = supersetImportedDocument() as { widgets: Record[] }; + raw.widgets[1].layout = { x: 9, y: 0, w: 6, h: 3 }; + expect(() => parseDashboardDocument(raw)).toThrow(/overflows the 12-column grid/); + }); + + it('an interval refresh with a cron, and a manual refresh with a schedule, are both refused', () => { + const a = supersetImportedDocument() as Record; + a.refresh = { mode: 'interval', interval_seconds: 300, cron: '0 * * * *', max_staleness_seconds: 900 }; + expect(() => parseDashboardDocument(a)).toThrow(/meaningless/); + + const b = supersetImportedDocument() as Record; + b.refresh = { mode: 'manual', interval_seconds: 300, max_staleness_seconds: 900 }; + expect(() => parseDashboardDocument(b)).toThrow(/manual/); + }); + + it('visibility "roles" with no roles is refused rather than guessed', () => { + const raw = supersetImportedDocument() as Record; + raw.permissions = { visibility: 'roles', roles: [] }; + expect(() => parseDashboardDocument(raw)).toThrow(/deny all/); + }); + + it('a required parameter that was not supplied fails the plan, not the render', () => { + const doc = parseDashboardDocument(supersetImportedDocument()); + expect(() => planDashboard(doc, { parameters: {} })).toThrow(/required parameter "since"/); + }); + + it('a locked filter cannot be switched off by a viewer', () => { + const doc = parseDashboardDocument(supersetImportedDocument()); + expect(() => + planDashboard(doc, { parameters: { since: '2026-01-01' }, disabled_filter_ids: ['f_tenant_scope'] }), + ).toThrow(/locked/); + // An unlocked one can be, and disappears from the compiled request. + const plan = planDashboard(doc, { + parameters: { since: '2026-01-01' }, + disabled_filter_ids: ['f_channel'], + }); + const table = plan.widgets.find((w) => w.widget_id === 'w_top_products'); + expect(table?.request.filters.map((f) => f.dimension)).not.toContain('channel'); + }); + + it('an enum parameter without allowed_values is refused', () => { + const raw = supersetImportedDocument() as { parameters: Record[] }; + raw.parameters[0] = { name: 'region', type: 'enum', required: false, default_value: 'SE' }; + expect(() => parseDashboardDocument(raw)).toThrow(/unbounded enum/); + }); +}); + +/** Rebuild an object graph with every object's keys in reverse order. */ +function shuffleKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(shuffleKeys); + if (value !== null && typeof value === 'object') { + const out: Record = {}; + for (const k of Object.keys(value as object).reverse()) { + out[k] = shuffleKeys((value as Record)[k]); + } + return out; + } + return value; +} + +describe('BI-P060 dashboard model — canonical hash', () => { + function parsed(): DashboardDocument { + return parseDashboardDocument(supersetImportedDocument()); + } + + it('is stable across key order and vendor re-export', () => { + const a = parsed(); + const b = parsed(); + b.vendor_ref = { system: 'superset', id: 'dashboard-999-after-migration' }; + b.widgets[0].vendor_ref = { system: 'metabase', id: 'card-3' }; + expect(dashboardDocumentHash(b)).toBe(dashboardDocumentHash(a)); + + // Key order in the source JSON must not change identity either. + expect(dashboardDocumentHash(parseDashboardDocument(shuffleKeys(supersetImportedDocument())))).toBe( + dashboardDocumentHash(a), + ); + }); + + it('changes when meaning changes', () => { + const base = dashboardDocumentHash(parsed()); + + const retitled = parsed(); + retitled.widgets[0].title = 'Net revenue'; + expect(dashboardDocumentHash(retitled)).not.toBe(base); + + const remetric = parsed(); + (remetric.widgets[0].source as { metric: string }).metric = 'revenue_net'; + expect(dashboardDocumentHash(remetric)).not.toBe(base); + + const moved = parsed(); + moved.widgets[1].layout = { x: 3, y: 1, w: 9, h: 4 }; + expect(dashboardDocumentHash(moved)).not.toBe(base); + + const refiltered = parsed(); + refiltered.filters[1].default_values = ['store']; + expect(dashboardDocumentHash(refiltered)).not.toBe(base); + }); +});