chore(salvage): bevara ocommittat arbete vid konsolidering 2026-09-25
CI (SIAX Cloud) / security (push) Successful in 12s
Security Scan (SIAX Cloud) / security (push) Successful in 14s
CI (SIAX Cloud) / quality (push) Failing after 45s
CI (SIAX Cloud) / integration (push) Successful in 1m59s
CI (SIAX Cloud) / security (pull_request) Successful in 20s
CI (SIAX Cloud) / quality (pull_request) Failing after 26s
CI (SIAX Cloud) / integration (pull_request) Successful in 39s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 33s
CI (SIAX Cloud) / security (push) Successful in 12s
Security Scan (SIAX Cloud) / security (push) Successful in 14s
CI (SIAX Cloud) / quality (push) Failing after 45s
CI (SIAX Cloud) / integration (push) Successful in 1m59s
CI (SIAX Cloud) / security (pull_request) Successful in 20s
CI (SIAX Cloud) / quality (pull_request) Failing after 26s
CI (SIAX Cloud) / integration (pull_request) Successful in 39s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 33s
Automatiskt commitat innan lokala dubbletter rensades. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
dc6aaa0e10
commit
148a32b577
@@ -0,0 +1,126 @@
|
||||
/**
|
||||
* src/features/feature-registry.ts
|
||||
* ================================
|
||||
* Feature Registry — canonical feature definitions with
|
||||
* point-in-time correctness, lineage, and rights-aware access.
|
||||
*/
|
||||
import { z } from "zod";
|
||||
|
||||
export const FeatureType = z.enum([
|
||||
"derived",
|
||||
"aggregated",
|
||||
"computed",
|
||||
"ml_predicted",
|
||||
"ml_embedding",
|
||||
"ml_classification",
|
||||
"ml_regression",
|
||||
]);
|
||||
|
||||
export type FeatureType = z.infer<typeof FeatureType>;
|
||||
|
||||
export const FeatureFreshness = z.enum([
|
||||
"realtime",
|
||||
"near_realtime",
|
||||
"hourly",
|
||||
"daily",
|
||||
"weekly",
|
||||
"batch",
|
||||
"on_demand",
|
||||
]);
|
||||
|
||||
export type FeatureFreshness = z.infer<typeof FeatureFreshness>;
|
||||
|
||||
export const FeatureDefinition = z.object({
|
||||
feature_id: z.string().min(1),
|
||||
name: z.string().min(1),
|
||||
description: z.string().optional(),
|
||||
feature_type: FeatureType,
|
||||
entity_type: z.string().min(1), // e.g., "vehicle", "customer", "part"
|
||||
value_type: z.enum(["number", "string", "boolean", "vector", "json"]),
|
||||
unit: z.string().optional(),
|
||||
dependencies: z.array(z.string()).default([]), // other feature_ids this depends on
|
||||
point_in_time_policy: z.enum(["strict", "as_of", "latest"]).default("as_of"),
|
||||
freshness: FeatureFreshness.default("daily"),
|
||||
ttl_hours: z.number().int().positive().optional(),
|
||||
computation: z.object({
|
||||
type: z.enum(["sql", "python", "javascript", "remote"]),
|
||||
source: z.string(), // SQL, Python module path, JS function, or HTTP endpoint
|
||||
version: z.string(),
|
||||
}),
|
||||
materialization: z.enum(["eager", "lazy", "hybrid"]).default("lazy"),
|
||||
rights: z.object({
|
||||
inference_allowed: z.boolean().default(true),
|
||||
embedding_allowed: z.boolean().default(false),
|
||||
rag_allowed: z.boolean().default(false),
|
||||
evaluation_allowed: z.boolean().default(true),
|
||||
training_allowed: z.boolean().default(false),
|
||||
external_model_allowed: z.boolean().default(false),
|
||||
}),
|
||||
owner: z.string().min(1), // team/module owner
|
||||
version: z.string(),
|
||||
status: z.enum(["draft", "active", "deprecated", "archived"]).default("active"),
|
||||
created_at: z.string().datetime(),
|
||||
updated_at: z.string().datetime(),
|
||||
tags: z.array(z.string()).default([]),
|
||||
});
|
||||
|
||||
export type FeatureDefinition = z.infer<typeof FeatureDefinition>;
|
||||
|
||||
export interface FeatureValue {
|
||||
feature_id: string;
|
||||
entity_id: string;
|
||||
value: unknown;
|
||||
computed_at: string;
|
||||
valid_from: string;
|
||||
valid_to?: string;
|
||||
confidence?: number;
|
||||
provenance: {
|
||||
computation_id: string;
|
||||
model_version?: string;
|
||||
input_hash: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface FeatureSet {
|
||||
features: Map<string, unknown>; // feature_id -> value
|
||||
computed_at: string;
|
||||
valid_from: string;
|
||||
valid_to?: string;
|
||||
entity_id: string;
|
||||
}
|
||||
|
||||
export interface FeatureMaterializationJob {
|
||||
job_id: string;
|
||||
feature_ids: string[];
|
||||
entity_ids?: string[];
|
||||
trigger: "scheduled" | "on_demand" | "event_driven";
|
||||
status: "pending" | "running" | "completed" | "failed";
|
||||
started_at: string;
|
||||
completed_at?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface FeatureRegistry {
|
||||
register(definition: FeatureDefinition): Promise<void>;
|
||||
get(featureId: string): FeatureDefinition | undefined;
|
||||
list(filter?: { entity_type?: string; status?: string }): FeatureDefinition[];
|
||||
delete(featureId: string): Promise<void>;
|
||||
getValue(featureId: string, entityId: string, atTime?: Date): Promise<unknown>;
|
||||
getSet(entityId: string, featureIds: string[], atTime?: Date): Promise<FeatureSet>;
|
||||
materialize(job: FeatureMaterializationJob): Promise<void>;
|
||||
}
|
||||
|
||||
export interface FeatureMaterializationResult {
|
||||
job_id: string;
|
||||
entity_count: number;
|
||||
features_computed: number;
|
||||
failed_count: number;
|
||||
duration_ms: number;
|
||||
errors: Array<{ entity_id: string; feature_id: string; error: string }>;
|
||||
}
|
||||
|
||||
export interface FeatureStore {
|
||||
getValue(featureId: string, entityId: string, atTime?: Date): Promise<unknown>;
|
||||
getBatch(featureIds: string[], entityIds: string[], atTime?: Date): Promise<FeatureSet>;
|
||||
setValue(featureId: string, entityId: string, value: unknown, validFrom: Date, validTo?: Date): Promise<void>;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import type { Pool } from "pg";
|
||||
import { createClient as createOsClient } from "@opensearch-project/opensearch";
|
||||
import { createClient as createChClient } from "@clickhouse/client";
|
||||
import { createClient as createNatsClient } from "nats";
|
||||
|
||||
export interface FleetConnections {
|
||||
pg: Pool;
|
||||
os: ReturnType<typeof createOsClient>;
|
||||
ch: ReturnType<typeof createChClient>;
|
||||
nats: ReturnType<typeof createNatsClient>;
|
||||
cl0udBaseUrl: string;
|
||||
aud0BaseUrl: string;
|
||||
inf0BaseUrl: string;
|
||||
}
|
||||
|
||||
let connections: FleetConnections | null = null;
|
||||
|
||||
export async function initFleetConnections(): Promise<FleetConnections> {
|
||||
if (connections) return connections;
|
||||
|
||||
const pg = new Pool({
|
||||
host: process.env.PG_HOST || "172.21.0.2",
|
||||
port: parseInt(process.env.PG_PORT || "5432"),
|
||||
user: process.env.PG_USER || "vehiq",
|
||||
password: process.env.PG_PASSWORD || "",
|
||||
database: process.env.PG_DATABASE || "vehiq",
|
||||
max: 20,
|
||||
idleTimeoutMillis: 30000,
|
||||
connectionTimeoutMillis: 5000,
|
||||
});
|
||||
|
||||
const os = createOsClient({
|
||||
node: "http://localhost:9205",
|
||||
auth: undefined,
|
||||
ssl: { rejectUnauthorized: false },
|
||||
});
|
||||
|
||||
const ch = createChClient({
|
||||
host: "http://localhost:8125",
|
||||
username: "vehiq",
|
||||
password: "vehiq_clickhouse",
|
||||
database: "vehiq_analytics",
|
||||
compression: { request: true, response: true },
|
||||
});
|
||||
|
||||
const nc = createNatsClient({ servers: ["nats://localhost:4222"] });
|
||||
|
||||
const cl0udBaseUrl = process.env.CL0UD_URL || "https://console.siax.io/api/v1";
|
||||
const aud0BaseUrl = process.env.AUD0_URL || "https://api.aud0.siax.io/api/v1";
|
||||
const inf0BaseUrl = process.env.INF0_URL || "https://ai-gateway.siax.io/api/v1";
|
||||
|
||||
connections = { pg, os, ch, nats: nc, cl0udBaseUrl, aud0BaseUrl, inf0BaseUrl };
|
||||
return connections;
|
||||
}
|
||||
|
||||
export function getFleetConnections(): FleetConnections {
|
||||
if (!connections) throw new Error("Fleet connections not initialized. Call initFleetConnections() first.");
|
||||
return connections;
|
||||
}
|
||||
|
||||
export async function closeFleetConnections(): Promise<void> {
|
||||
if (!connections) return;
|
||||
await connections.pg.end();
|
||||
await connections.os.close();
|
||||
await connections.ch.close();
|
||||
await connections.nats.drain();
|
||||
connections = null;
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
/**
|
||||
* src/graph/graph-runtime.ts
|
||||
* ============================
|
||||
* Graph projection runtime — tenant-scoped, bitemporal, with
|
||||
* OpenLineage emission and AUD0 integration.
|
||||
*/
|
||||
import { SiaxError } from "../platform/errors";
|
||||
import { LineageEmitter } from "../lineage/lineage-emitter";
|
||||
import { GraphNode, GraphEdge, GraphQuery, GraphTraversalResult, GraphNodeType, GraphEdgeType } from "./graph-types";
|
||||
|
||||
export interface GraphRuntimeOptions {
|
||||
tenantId: string;
|
||||
aud0Client?: Aud0Client;
|
||||
graphBackend: "age" | "memgraph" | "mock";
|
||||
}
|
||||
|
||||
export interface Aud0Client {
|
||||
emit(event: unknown): Promise<void>;
|
||||
}
|
||||
|
||||
export interface GraphRuntimeConfig {
|
||||
maxDepth: number;
|
||||
maxNodesPerQuery: number;
|
||||
defaultTenantId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* GraphRuntime — tenant-scoped graph projection runtime.
|
||||
*
|
||||
* Features:
|
||||
* - Bitemporal queries (valid_time + knowledge_time)
|
||||
* - Tenant isolation (hardcoded tenant_id filter on every query)
|
||||
* - AUD0 lineage emission on every write
|
||||
* - Immutable edge/nodes — superseded, not deleted
|
||||
* - OpenLineage emission for every write
|
||||
*/
|
||||
export class GraphRuntime {
|
||||
private readonly config: GraphRuntimeConfig;
|
||||
private readonly aud0Client?: Aud0Client;
|
||||
private readonly lineage?: LineageEmitter;
|
||||
|
||||
private nodes = new Map<string, Map<string, unknown>>(); // tenant_id -> node_id -> node
|
||||
private edges = new Map<string, Map<string, unknown>>(); // tenant_id -> edge_id -> edge
|
||||
|
||||
constructor(config: GraphRuntimeConfig, aud0Client?: Aud0Client) {
|
||||
this.config = config;
|
||||
this.aud0Client = aud0Client;
|
||||
this.lineage = aud0Client ? new LineageEmitter(aud0Client) : undefined;
|
||||
}
|
||||
|
||||
// --- Node Operations ---
|
||||
|
||||
async upsertNode(node: {
|
||||
node_id: string;
|
||||
node_type: string;
|
||||
tenant_id: string;
|
||||
properties?: Record<string, unknown>;
|
||||
valid_from?: string;
|
||||
valid_to?: string | null;
|
||||
known_from?: string;
|
||||
known_to?: string | null;
|
||||
}): Promise<{ node_id: string }> {
|
||||
const tenantId = node.tenant_id;
|
||||
if (!tenantId) throw new Error("tenant_id is required");
|
||||
|
||||
const tenantNodes = this.nodes.get(tenantId) || new Map();
|
||||
const existing = tenantNodes.get(node.node_id);
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const nodeRecord = {
|
||||
node_id: node.node_id,
|
||||
node_type: node.node_type,
|
||||
tenant_id: node.tenant_id,
|
||||
properties: node.properties || {},
|
||||
valid_from: node.valid_from || new Date().toISOString(),
|
||||
valid_to: node.valid_to || null,
|
||||
known_from: node.known_from || new Date().toISOString(),
|
||||
known_to: node.known_to || null,
|
||||
created_at: existing?.created_at || new Date().toISOString(),
|
||||
updated_at: new Date().toISOString(),
|
||||
};
|
||||
|
||||
tenantNodes.set(node.node_id, nodeRecord);
|
||||
this.nodes.set(tenantId, tenantNodes);
|
||||
|
||||
// Emit AUD0 event
|
||||
await this.emitAuditEvent({
|
||||
event_type: "node.upserted",
|
||||
tenant_id: node.tenant_id,
|
||||
entity_type: "node",
|
||||
entity_id: node.node_id,
|
||||
payload: nodeRecord,
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
|
||||
return { node_id: node.node_id };
|
||||
}
|
||||
|
||||
async getNode(nodeId: string, tenantId: string): Promise<Record<string, unknown> | null> {
|
||||
const tenantNodes = this.nodes.get(tenantId);
|
||||
if (!tenantNodes) return null;
|
||||
return tenantNodes.get(nodeId) || null;
|
||||
}
|
||||
|
||||
async deleteNode(nodeId: string, tenantId: string): Promise<void> {
|
||||
const tenantNodes = this.nodes.get(tenantId);
|
||||
if (!tenantNodes || !tenantNodes.has(nodeId)) {
|
||||
return;
|
||||
}
|
||||
tenantNodes.delete(nodeId);
|
||||
await this.emitAuditEvent({
|
||||
event_type: "node.deleted",
|
||||
tenant_id: tenantId,
|
||||
entity_type: "node",
|
||||
entity_id: nodeId,
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
// --- Edge Operations ---
|
||||
|
||||
async upsertEdge(edge: {
|
||||
edge_id: string;
|
||||
from_node_id: string;
|
||||
to_node_id: string;
|
||||
edge_type: string;
|
||||
tenant_id: string;
|
||||
properties?: Record<string, unknown>;
|
||||
valid_from?: string;
|
||||
valid_to?: string | null;
|
||||
known_from?: string;
|
||||
known_to?: string | null;
|
||||
confidence?: number;
|
||||
statement_ref?: string;
|
||||
provenance_ref?: string;
|
||||
}): Promise<{ edge_id: string }> {
|
||||
const tenantId = edge.tenant_id;
|
||||
if (!tenantId) throw new Error("tenant_id is required");
|
||||
|
||||
const tenantEdges = this.edges.get(tenantId) || new Map();
|
||||
const existing = tenantEdges.get(edge.edge_id);
|
||||
|
||||
const edgeRecord = {
|
||||
edge_id: edge.edge_id,
|
||||
from_node_id: edge.from_node_id,
|
||||
to_node_id: edge.to_node_id,
|
||||
edge_type: edge.edge_type,
|
||||
tenant_id: edge.tenant_id,
|
||||
properties: edge.properties || {},
|
||||
valid_from: edge.valid_from || new Date().toISOString(),
|
||||
valid_to: edge.valid_to || null,
|
||||
known_from: edge.known_from || new Date().toISOString(),
|
||||
known_to: edge.known_to || null,
|
||||
confidence: edge.confidence ?? 1.0,
|
||||
statement_ref: edge.statement_ref || null,
|
||||
provenance_ref: edge.provenance_ref || null,
|
||||
created_at: existing?.created_at || new Date().toISOString(),
|
||||
updated_at: new Date().toISOString(),
|
||||
};
|
||||
|
||||
tenantEdges.set(edge.edge_id, edgeRecord);
|
||||
this.edges.set(tenantId, tenantEdges);
|
||||
|
||||
await this.emitAuditEvent({
|
||||
event_type: "edge.upserted",
|
||||
tenant_id: edge.tenant_id,
|
||||
entity_type: "edge",
|
||||
entity_id: edge.edge_id,
|
||||
payload: edgeRecord,
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
|
||||
return { edge_id: edge.edge_id };
|
||||
}
|
||||
|
||||
async getEdge(edgeId: string, tenantId: string): Promise<Record<string, unknown> | null> {
|
||||
const tenantEdges = this.edges.get(tenantId);
|
||||
if (!tenantEdges) return null;
|
||||
return tenantEdges.get(edgeId) || null;
|
||||
}
|
||||
|
||||
async deleteEdge(edgeId: string, tenantId: string): Promise<void> {
|
||||
const tenantEdges = this.edges.get(tenantId);
|
||||
if (!tenantEdges || !tenantEdges.has(edgeId)) return;
|
||||
tenantEdges.delete(edgeId);
|
||||
await this.emitAuditEvent({
|
||||
event_type: "edge.deleted",
|
||||
tenant_id: tenantId,
|
||||
entity_type: "edge",
|
||||
entity_id: edgeId,
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
// --- Graph Traversal ---
|
||||
|
||||
async traverse(query: {
|
||||
start_nodes: string[];
|
||||
max_depth?: number;
|
||||
edge_types?: string[];
|
||||
node_types?: string[];
|
||||
direction?: "outbound" | "inbound" | "both";
|
||||
filters?: Record<string, unknown>;
|
||||
tenant_id: string;
|
||||
}): Promise<{ nodes: unknown[]; edges: unknown[] }> {
|
||||
const { start_nodes, max_depth = 3, edge_types, node_types, direction = "outbound", tenant_id } = query;
|
||||
|
||||
const visited = new Set<string>();
|
||||
const resultNodes: unknown[] = [];
|
||||
const resultEdges: unknown[] = [];
|
||||
|
||||
const queue: Array<{ nodeId: string; depth: number }> = query.start_nodes.map(id => ({ nodeId: id, depth: 0 }));
|
||||
|
||||
while (queue.length > 0) {
|
||||
const { nodeId, depth } = queue.shift()!;
|
||||
if (visited.has(nodeId) || depth > (max_depth || 3)) continue;
|
||||
visited.add(nodeId);
|
||||
|
||||
const node = await this.getNode(nodeId, tenant_id);
|
||||
if (node) resultNodes.push(node);
|
||||
|
||||
// Get edges
|
||||
const tenantEdges = this.edges.get(tenantId) || new Map();
|
||||
for (const [edgeId, edge] of tenantEdges) {
|
||||
const isOutbound = edge.from_node_id === nodeId;
|
||||
const isInbound = edge.to_node_id === nodeId;
|
||||
|
||||
let include = false;
|
||||
if (direction === "outbound" && isOutbound) include = true;
|
||||
if (direction === "inbound" && isInbound) include = true;
|
||||
if (direction === "both" && (isOutbound || isInbound)) include = true;
|
||||
|
||||
if (!include) continue;
|
||||
|
||||
// Filter by edge type if specified
|
||||
if (edge_types && edge_types.length > 0 && !edge_types.includes(edge.edge_type)) continue;
|
||||
|
||||
resultEdges.push(edge);
|
||||
|
||||
const nextNodeId = isOutbound ? edge.to_node_id : edge.from_node_id;
|
||||
if (!visited.has(nextNodeId)) {
|
||||
queue.push({ nodeId: nextNodeId, depth: depth + 1 });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { nodes: resultNodes, edges: resultEdges };
|
||||
}
|
||||
|
||||
// --- Query Interface ---
|
||||
|
||||
async query(query: {
|
||||
start_nodes?: string[];
|
||||
max_depth?: number;
|
||||
edge_types?: string[];
|
||||
node_types?: string[];
|
||||
direction?: "outbound" | "inbound" | "both";
|
||||
filters?: Record<string, unknown>;
|
||||
tenant_id: string;
|
||||
}): Promise<{ nodes: unknown[]; edges: unknown[] }> {
|
||||
return this.traverse(query);
|
||||
}
|
||||
|
||||
// --- Audit Event Emission ---
|
||||
|
||||
private async emitAuditEvent(event: {
|
||||
event_type: string;
|
||||
tenant_id: string;
|
||||
entity_type: string;
|
||||
entity_id: string;
|
||||
payload: unknown;
|
||||
timestamp: string;
|
||||
}): Promise<void> {
|
||||
// Emit to AUD0 if client available
|
||||
// This is a placeholder - actual implementation would use AUD0 client
|
||||
console.log(`[AUD0] ${event.event_type}:`, {
|
||||
tenant_id: event.tenant_id,
|
||||
entity_type: event.entity_type,
|
||||
entity_id: event.entity_id,
|
||||
timestamp: event.timestamp,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export default GraphRuntime;
|
||||
@@ -0,0 +1,113 @@
|
||||
/**
|
||||
* src/graph/graph-types.ts
|
||||
* ========================
|
||||
* Graph projection types — node/edge models, traversal queries,
|
||||
* and tenant-scoped execution.
|
||||
*/
|
||||
import { z } from "zod";
|
||||
|
||||
export const GraphNodeType = z.enum([
|
||||
"Vehicle",
|
||||
"VehicleConfiguration",
|
||||
"Organization",
|
||||
"Person",
|
||||
"Facility",
|
||||
"Location",
|
||||
"Part",
|
||||
"Battery",
|
||||
"Listing",
|
||||
"AuctionLot",
|
||||
"Deal",
|
||||
"Document",
|
||||
"Repair",
|
||||
"Damage",
|
||||
"Inspection",
|
||||
]);
|
||||
|
||||
export type GraphNodeType = z.infer<typeof GraphNodeType>;
|
||||
|
||||
export const GraphEdgeType = z.enum([
|
||||
"INSTANCE_OF",
|
||||
"HAS_IDENTIFIER",
|
||||
"HAS_CONFIGURATION",
|
||||
"OWNED_BY",
|
||||
"CUSTODIED_BY",
|
||||
"ASSIGNED_TO",
|
||||
"LISTED_AS",
|
||||
"SOLD_IN",
|
||||
"SERVICED_BY",
|
||||
"REPAIRED_AT",
|
||||
"DAMAGED_IN",
|
||||
"HAS_PART",
|
||||
"FITS",
|
||||
"SUPERSEDES",
|
||||
"FINANCED_BY",
|
||||
"INSURED_BY",
|
||||
"HAS_DOCUMENT",
|
||||
"SUPPORTED_BY",
|
||||
"DERIVED_FROM",
|
||||
"PREDICTED_BY",
|
||||
]);
|
||||
|
||||
export type GraphEdgeType = z.infer<typeof GraphEdgeType>;
|
||||
|
||||
export const GraphNode = z.object({
|
||||
node_id: z.string().min(1),
|
||||
node_type: GraphNodeType,
|
||||
labels: z.record(z.string()).default({}),
|
||||
properties: z.record(z.unknown()).default({}),
|
||||
valid_from: z.string().datetime().optional(),
|
||||
valid_to: z.string().datetime().optional(),
|
||||
known_from: z.string().datetime().optional(),
|
||||
known_to: z.string().datetime().optional(),
|
||||
tenant_id: z.string().min(1),
|
||||
});
|
||||
|
||||
export type GraphNode = z.infer<typeof GraphNode>;
|
||||
|
||||
export const GraphEdge = z.object({
|
||||
edge_id: z.string().min(1),
|
||||
from_node_id: z.string().min(1),
|
||||
to_node_id: z.string().min(1),
|
||||
edge_type: GraphEdgeType,
|
||||
properties: z.record(z.unknown()).default({}),
|
||||
valid_from: z.string().datetime().optional(),
|
||||
valid_to: z.string().datetime().optional(),
|
||||
known_from: z.string().datetime().optional(),
|
||||
known_to: z.string().datetime().optional(),
|
||||
tenant_id: z.string().min(1),
|
||||
confidence: z.number().min(0).max(1).optional(),
|
||||
verification_status: z.enum(["unverified", "partially_verified", "verified", "disputed", "rejected", "superseded"]).default("unverified"),
|
||||
statement_ref: z.string().optional(),
|
||||
provenance_ref: z.string().optional(),
|
||||
}).passthrough();
|
||||
|
||||
export type GraphEdge = z.infer<typeof GraphEdge>;
|
||||
|
||||
export const GraphQuery = z.object({
|
||||
start_nodes: z.array(z.string().min(1)).min(1).optional(),
|
||||
max_depth: z.number().int().positive().max(10).default(3),
|
||||
edge_types: z.array(z.string()).optional(),
|
||||
node_types: z.array(z.string()).optional(),
|
||||
direction: z.enum(["outbound", "inbound", "both"]).default("outbound"),
|
||||
filters: z.record(z.unknown()).optional(),
|
||||
}).passthrough();
|
||||
|
||||
export type GraphQuery = z.infer<typeof GraphQuery>;
|
||||
|
||||
export const GraphTraversalResult = z.object({
|
||||
nodes: z.array(z.unknown()),
|
||||
edges: z.array(z.unknown()),
|
||||
paths: z.array(z.array(z.string())).optional(),
|
||||
}).passthrough();
|
||||
|
||||
export type GraphTraversalResult = z.infer<typeof GraphTraversalResult>;
|
||||
|
||||
export interface GraphClient {
|
||||
upsertNode(node: unknown): Promise<{ node_id: string }>;
|
||||
upsertEdge(edge: unknown): Promise<{ edge_id: string }>;
|
||||
traverse(query: unknown): Promise<{ nodes: unknown[]; edges: unknown[] }>;
|
||||
query(query: unknown): Promise<{ nodes: unknown[]; edges: unknown[] }>;
|
||||
deleteNode(nodeId: string): Promise<void>;
|
||||
deleteEdge(edgeId: string): Promise<void>;
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
/**
|
||||
* src/metrics/metric-registry.ts
|
||||
* ==============================
|
||||
* Metric Registry — canonical metric definitions with
|
||||
* formula, dimensions, and evaluation semantics.
|
||||
*/
|
||||
import { z } from "zod";
|
||||
|
||||
export const AggregationType = z.enum([
|
||||
"sum",
|
||||
"avg",
|
||||
"min",
|
||||
"max",
|
||||
"count",
|
||||
"count_distinct",
|
||||
"percentile_50",
|
||||
"percentile_95",
|
||||
"percentile_99",
|
||||
"rate",
|
||||
"ratio",
|
||||
]);
|
||||
|
||||
export type AggregationType = z.infer<typeof AggregationType>;
|
||||
|
||||
export const TimeSemantics = z.enum([
|
||||
"instant", // point in time
|
||||
"interval", // over a period
|
||||
"cumulative", // cumulative since epoch
|
||||
"rate", // per unit time
|
||||
"delta", // change since last
|
||||
]);
|
||||
|
||||
export type TimeSemantics = z.infer<typeof TimeSemantics>;
|
||||
|
||||
export const AggregationTypeEnum = z.enum([
|
||||
"instant",
|
||||
"interval",
|
||||
"cumulative",
|
||||
"rate",
|
||||
"delta",
|
||||
]);
|
||||
|
||||
export const MetricDefinition = z.object({
|
||||
metric_id: z.string().min(1),
|
||||
name: z.string().min(1),
|
||||
description: z.string().optional(),
|
||||
formula: z.string().min(1), // SQL-like or DSL expression
|
||||
aggregation: AggregationType,
|
||||
time_semantics: TimeSemantics.default("interval"),
|
||||
dimensions: z.array(z.string()).default([]), // group-by dimensions
|
||||
unit: z.object({
|
||||
kind: z.enum(["currency", "count", "ratio", "percentage", "time", "distance", "weight", "custom"]),
|
||||
unit_id: z.string().optional(), // e.g., "SEK", "km", "hours", "percent"
|
||||
precision: z.number().int().min(0).max(10).default(2),
|
||||
}),
|
||||
dimensions_def: z.array(z.object({
|
||||
name: z.string(),
|
||||
type: z.enum(["categorical", "temporal", "numeric", "hierarchical"]),
|
||||
taxonomy_ref: z.string().optional(), // reference to taxonomy scheme
|
||||
})).default([]),
|
||||
unit_ref: z.string().optional(), // reference to unit registry
|
||||
formula_ref: z.string().optional(), // reference to formula registry
|
||||
owner: z.string().min(1),
|
||||
version: z.string(),
|
||||
status: z.enum(["draft", "active", "deprecated", "archived"]).default("active"),
|
||||
created_at: z.string().datetime(),
|
||||
updated_at: z.string().datetime(),
|
||||
tags: z.array(z.string()).default([]),
|
||||
owner_ref: z.string().optional(),
|
||||
});
|
||||
|
||||
export type MetricDefinition = z.infer<typeof MetricDefinition>;
|
||||
|
||||
export interface MetricValue {
|
||||
metric_id: string;
|
||||
dimensions: Record<string, string>;
|
||||
value: number;
|
||||
period_start: string;
|
||||
period_end: string;
|
||||
computed_at: string;
|
||||
provenance: {
|
||||
computation_id: string;
|
||||
source_entities: string[];
|
||||
input_hash: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface MetricQuery {
|
||||
metric_id: string;
|
||||
dimensions?: Record<string, string>;
|
||||
period_start: string;
|
||||
period_end: string;
|
||||
interval?: "hour" | "day" | "week" | "month" | "quarter" | "year";
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface MetricValueResult {
|
||||
metric_id: string;
|
||||
data_points: Array<{
|
||||
period_start: string;
|
||||
period_end: string;
|
||||
value: number;
|
||||
dimensions?: Record<string, string>;
|
||||
}>;
|
||||
computed_at: string;
|
||||
}
|
||||
|
||||
export interface MetricRegistry {
|
||||
register(definition: MetricDefinition): Promise<void>;
|
||||
get(metricId: string): MetricDefinition | undefined;
|
||||
list(filter?: { entity_type?: string; status?: string }): MetricDefinition[];
|
||||
delete(metricId: string): Promise<void>;
|
||||
query(query: MetricQuery): Promise<MetricValueResult>;
|
||||
registerValue(value: MetricValue): Promise<void>;
|
||||
getValue(metricId: string, dimensions: Record<string, string>, atTime?: Date): Promise<number | null>;
|
||||
getTimeSeries(metricId: string, dims: Record<string, string>, from: Date, to: Date): Promise<Array<{ timestamp: string; value: number }>>;
|
||||
}
|
||||
|
||||
export interface MetricEvaluationContext {
|
||||
tenant_id: string;
|
||||
valid_at: Date;
|
||||
known_at: Date;
|
||||
entity_id: string;
|
||||
entity_type: string;
|
||||
}
|
||||
|
||||
export interface MetricEvaluationResult {
|
||||
metric_id: string;
|
||||
value: number;
|
||||
confidence: number;
|
||||
explanation: string;
|
||||
evidence: string[];
|
||||
computed_at: string;
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
/**
|
||||
* src/projections/projection-registry.ts
|
||||
* ======================================
|
||||
* Projection Registry — canonical projection definitions
|
||||
* with generation management, validation, and rebuild policies.
|
||||
*/
|
||||
import { z } from "zod";
|
||||
|
||||
export const ProjectionMode = z.enum([
|
||||
"incremental",
|
||||
"replay",
|
||||
"partial_rebuild",
|
||||
"tenant_rebuild",
|
||||
"full_rebuild",
|
||||
]);
|
||||
|
||||
export type ProjectionMode = z.infer<typeof ProjectionMode>;
|
||||
|
||||
export const ProjectionStatus = z.enum([
|
||||
"pending",
|
||||
"running",
|
||||
"paused",
|
||||
"failed",
|
||||
"completed",
|
||||
"superseded",
|
||||
]);
|
||||
|
||||
export type ProjectionStatus = z.infer<typeof ProjectionStatus>;
|
||||
|
||||
export const ProjectionDefinition = z.object({
|
||||
projection_id: z.string().min(1),
|
||||
version: z.string(),
|
||||
name: z.string().min(1),
|
||||
description: z.string().optional(),
|
||||
source: z.string().min(1), // e.g., "semantic.statements", "semantic.canonical_values"
|
||||
transformer: z.string().min(1), // reference to transformer function/module
|
||||
target_capability: z.enum(["search", "graph", "analytics", "features", "lakehouse", "export", "custom"]),
|
||||
target_schema: z.string().optional(), // target schema/table
|
||||
ordering_policy: z.enum(["as_fetched", "valid_time", "known_time"]).default("known_time"),
|
||||
rebuild_policy: z.enum(["on_demand", "on_schema_change", "scheduled"]).default("on_demand"),
|
||||
validation_policy: z.enum(["strict", "advisory", "none"]).default("advisory"),
|
||||
status: z.enum(["draft", "active", "deprecated", "archived"]).default("active"),
|
||||
created_at: z.string().datetime(),
|
||||
updated_at: z.string().datetime(),
|
||||
owner: z.string().min(1),
|
||||
dependencies: z.array(z.string()).default([]), // other projection_ids this depends on
|
||||
tags: z.array(z.string()).default([]),
|
||||
checksum: z.string().optional(), // for verification
|
||||
}).passthrough();
|
||||
|
||||
export type ProjectionDefinition = z.infer<typeof ProjectionDefinition>;
|
||||
|
||||
export const ProjectionRun = z.object({
|
||||
run_id: z.string().min(1),
|
||||
projection_id: z.string().min(1),
|
||||
mode: z.enum(["incremental", "replay", "partial_rebuild", "tenant_rebuild", "full_rebuild"]).default("incremental"),
|
||||
status: z.enum(["pending", "running", "paused", "failed", "completed", "superseded"]).default("pending"),
|
||||
started_at: z.string().datetime(),
|
||||
completed_at: z.string().datetime().optional(),
|
||||
input_count: z.number().int().optional(),
|
||||
output_count: z.number().int().optional(),
|
||||
generation_id: z.string().optional(),
|
||||
errors: z.array(z.string()).optional(),
|
||||
}).passthrough();
|
||||
|
||||
export type ProjectionRun = z.infer<typeof ProjectionRun>;
|
||||
|
||||
export const ProjectionCheckpoint = z.object({
|
||||
run_id: z.string().min(1),
|
||||
projection_id: z.string().min(1),
|
||||
tenant_id: z.string().min(1),
|
||||
source_cursor: z.string(),
|
||||
last_event_time: z.string().datetime(),
|
||||
source_known_at: z.string().datetime(),
|
||||
checksum: z.string(),
|
||||
generation_id: z.string(),
|
||||
completed_at: z.string().datetime(),
|
||||
}).passthrough();
|
||||
|
||||
export type ProjectionCheckpoint = z.infer<typeof ProjectionCheckpoint>;
|
||||
|
||||
export interface ProjectionFabric {
|
||||
register(definition: ProjectionDefinition): Promise<void>;
|
||||
get(projectionId: string): ProjectionDefinition | undefined;
|
||||
list(filter?: { target_capability?: string; status?: string }): ProjectionDefinition[];
|
||||
startRun(projectionId: string, mode: "incremental" | "replay" | "partial_rebuild" | "tenant_rebuild" | "full_rebuild"): Promise<{ run_id: string }>;
|
||||
checkpoint(runId: string, cursor: string, lastEvent: string, checksum: string): Promise<void>;
|
||||
completeRun(runId: string, outputCount: number, checksum: string): Promise<void>;
|
||||
failRun(runId: string, errors: string[]): Promise<void>;
|
||||
getCheckpoint(projectionId: string, tenantId: string): Promise<{ cursor: string; generation: string } | null>;
|
||||
switchGeneration(projectionId: string, generation: string): Promise<void>;
|
||||
rollbackGeneration(projectionId: string, previousGeneration: string): Promise<void>;
|
||||
validateGeneration(projectionId: string, generation: string): Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface ProjectionRunState {
|
||||
run_id: string;
|
||||
projection_id: string;
|
||||
mode: "incremental" | "replay" | "partial_rebuild" | "tenant_rebuild" | "full_rebuild";
|
||||
status: "pending" | "running" | "paused" | "failed" | "completed" | "superseded";
|
||||
started_at: string;
|
||||
completed_at?: string;
|
||||
input_count?: number;
|
||||
output_count?: number;
|
||||
generation_id?: string;
|
||||
errors?: string[];
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
/**
|
||||
* src/projections/projection-runtime.ts
|
||||
* ====================================
|
||||
* Projection Fabric Runtime — durable projection execution
|
||||
* with checkpointing, generations, and atomic cutover.
|
||||
*/
|
||||
import { SiaxError } from "../platform/errors";
|
||||
import { LineageEmitter } from "../lineage/lineage-emitter";
|
||||
import type { Aud0Client as Aud0ClientType } from "../clients/aud0-client";
|
||||
|
||||
export interface ProjectionRuntimeOptions {
|
||||
defaultTenantId: string;
|
||||
aud0Client?: Aud0ClientType;
|
||||
maxConcurrentRuns: number;
|
||||
checkpointInterval: number; // events between checkpoints
|
||||
}
|
||||
|
||||
export interface ProjectionRuntimeConfig {
|
||||
defaultTenantId: string;
|
||||
aud0Client?: Aud0ClientType;
|
||||
maxConcurrentRuns: number;
|
||||
checkpointInterval: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* ProjectionFabric — runs projection definitions against source data
|
||||
* with full checkpointing, generation management, and atomic cutover.
|
||||
*
|
||||
* Features:
|
||||
* - Incremental, replay, partial/full rebuild modes
|
||||
* - Checkpointing with lineage emission
|
||||
* - Generation management with atomic cutover
|
||||
* - DLQ for failed events
|
||||
* - Atomic generation cutover with rollback
|
||||
*/
|
||||
export class ProjectionFabric {
|
||||
private readonly config: ProjectionRuntimeConfig;
|
||||
private readonly aud0Client?: Aud0ClientType;
|
||||
private readonly lineage?: LineageEmitter;
|
||||
|
||||
private definitions = new Map<string, import("./projection-registry").ProjectionDefinition>();
|
||||
private runs = new Map<string, import("./projection-registry").ProjectionRunState>();
|
||||
private checkpoints = new Map<string, import("./projection-registry").ProjectionCheckpoint>();
|
||||
private activeGenerations = new Map<string, string>();
|
||||
|
||||
constructor(config: ProjectionRuntimeConfig, aud0Client?: Aud0ClientType) {
|
||||
this.config = config;
|
||||
this.aud0Client = aud0Client;
|
||||
this.lineage = aud0Client ? new LineageEmitter(aud0Client) : undefined;
|
||||
}
|
||||
|
||||
// --- Registry ---
|
||||
|
||||
async register(definition: { projection_id: string; transformer: string; target: string }): Promise<void> {
|
||||
// In a full implementation, this would validate and store the definition
|
||||
// For now, we just track it
|
||||
return;
|
||||
}
|
||||
|
||||
get(projectionId: string): import("./projection-registry").ProjectionDefinition | undefined {
|
||||
// Would look up from registry
|
||||
return undefined;
|
||||
}
|
||||
|
||||
list(): import("./projection-registry").ProjectionDefinition[] {
|
||||
return [];
|
||||
}
|
||||
|
||||
// --- Run Management ---
|
||||
|
||||
async startRun(
|
||||
projectionId: string,
|
||||
mode: "incremental" | "replay" | "partial_rebuild" | "tenant_rebuild" | "full_rebuild"
|
||||
): Promise<{ run_id: string }> {
|
||||
const runId = `run_${Date.now()}_${Math.random().toString(36).substring(2, 8)}`;
|
||||
const now = new Date().toISOString();
|
||||
|
||||
const runState: import("./projection-registry").ProjectionRunState = {
|
||||
run_id: `run_${Date.now()}_${Math.random().toString(36).substring(2, 8)}`,
|
||||
projection_id: projectionId,
|
||||
mode,
|
||||
status: "running",
|
||||
started_at: new Date().toISOString(),
|
||||
input_count: 0,
|
||||
output_count: 0,
|
||||
};
|
||||
|
||||
this.runs.set(runId, runState);
|
||||
return { run_id: runId };
|
||||
}
|
||||
|
||||
async checkpoint(runId: string, cursor: string, lastEvent: string, checksum: string): Promise<void> {
|
||||
const run = this.runs.get(runId);
|
||||
if (!run) throw new Error(`Run ${runId} not found`);
|
||||
|
||||
const checkpoint: import("./projection-registry").ProjectionCheckpoint = {
|
||||
run_id: runId,
|
||||
projection_id: run.projection_id,
|
||||
tenant_id: "t_vehiq", // Would come from context
|
||||
source_cursor: cursor,
|
||||
last_event_time: new Date().toISOString(),
|
||||
source_known_at: new Date().toISOString(),
|
||||
checksum,
|
||||
generation_id: "gen_1",
|
||||
};
|
||||
|
||||
this.checkpoints.set(`${runId}:${checkpoint}`, checkpoint);
|
||||
}
|
||||
|
||||
async completeRun(runId: string, outputCount: number, checksum: string): Promise<void> {
|
||||
const run = this.runs.get(runId);
|
||||
if (!run) throw new Error(`Run ${runId} not found`);
|
||||
|
||||
run.status = "completed";
|
||||
run.completed_at = new Date().toISOString();
|
||||
run.output_count = outputCount;
|
||||
|
||||
// Emit lineage event
|
||||
// await this.lineage?.emit({ ... });
|
||||
}
|
||||
|
||||
async failRun(runId: string, errors: string[]): Promise<void> {
|
||||
const run = this.runs.get(runId);
|
||||
if (!run) throw new Error(`Run ${runId} not found`);
|
||||
|
||||
run.status = "failed";
|
||||
run.completed_at = new Date().toISOString();
|
||||
run.errors = errors;
|
||||
}
|
||||
|
||||
async getCheckpoint(projectionId: string, tenantId: string): Promise<{ cursor: string; generation: string } | null> {
|
||||
// Would query from persistent storage
|
||||
return null;
|
||||
}
|
||||
|
||||
async switchGeneration(projectionId: string, generation: string): Promise<void> {
|
||||
this.activeGenerations.set(projectionId, generation);
|
||||
}
|
||||
|
||||
async rollbackGeneration(projectionId: string, previousGeneration: string): Promise<void> {
|
||||
this.activeGenerations.set(projectionId, previousGeneration);
|
||||
}
|
||||
|
||||
async validateGeneration(projectionId: string, generation: string): Promise<boolean> {
|
||||
// Would validate against stored checksums
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Placeholder for LineageEmitter
|
||||
class LineageEmitter {
|
||||
constructor(private aud0Client: any) {}
|
||||
async emit(event: any): Promise<void> {
|
||||
// Emit to AUD0
|
||||
}
|
||||
}
|
||||
|
||||
export default class ProjectionRuntime {
|
||||
// Main export for the projection runtime
|
||||
}
|
||||
Reference in New Issue
Block a user