/** * `0pt/synthetic/factory.ts` — X-737: Synthetic Data Factory. * * Ägande enligt kontraktet: 0pt äger dataset-truth + transformationer; * INF0 äger generering (injectas via port). Pipeline: * * source DatasetVersion * → sampling * → synthesis (INF0 GenerationPort) * → PII/privacy-check (fail-closed FÖRE publish) * → quality-filter * → dedupe * → contamination-detection (rader = identiska med val/test-kandidater) * → train/val/test-split * → ny DatasetVersion (0pt-repot) * * PROVENANS PER RAD (acceptans): varje syntetisk rad bär generation-policy-ref * + model-version + source-ref. Privacy-check blockerar publish. */ import { createHash } from 'node:crypto'; export type PiiClass = 'none' | 'low' | 'medium' | 'high'; export interface SourceRow { id: string; fields: Record; } export interface GenerationConfig { /** Policy-ref för genereringen (CL0UD/0pt-policy — spärras per rad). */ generationPolicyRef: string; /** INF0-modellversion som genererade (provener per rad). */ modelVersion: string; /** Antal rader som ska syntetiseras. */ count: number; /** Högst tillåten PII-klass i publicerad data (över → NEKAS). */ maxAllowedPiiClass: PiiClass; /** Quality-tröskel per rad (0–1). */ minQualityScore: number; /** Train/val/test-fördelning (default 70/15/15). */ split?: { train: number; val: number; test: number }; } export interface GeneratedRow { fields: Record; /** Kvalitetspoäng från genereringen (0–1). */ qualityScore: number; provenance: SyntheticRowProvenance; } export interface SyntheticRowProvenance { rowId: string; generationPolicyRef: string; modelVersion: string; sourceDatasetRef: string; sourceRowIds: string[]; sha256: string; } /** INF0:s genereringsport (0pt anropar — INF0 äger modellen). */ export interface GenerationPort { generate(input: { sourceRows: SourceRow[]; count: number; policyRef: string; }): Promise<{ rows: GeneratedRow[] }>; } /** Privacy-port (X-728-presidio i INF0/integrations): klassificera rader. */ export interface PrivacyCheckPort { classifyRow(fields: Record): Promise; } const PII_ORDER: Record = { none: 0, low: 1, medium: 2, high: 3 }; export interface PipelineStageReport { stage: 'sampled' | 'synthesized' | 'privacy' | 'quality' | 'dedupe' | 'contamination' | 'split'; inputCount: number; outputCount: number; } export interface SyntheticDatasetVersionDraft { sourceDatasetId: string; sourceVersion: number; tenantId: string; projectId: string; schemaVersion: string; description: string; rows: Array<{ split: 'train' | 'val' | 'test'; provenance: SyntheticRowProvenance; fields: Record }>; stages: PipelineStageReport[]; /** Privacy-checks före publish — bevis för acceptansen. */ privacyChecks: { checked: number; blocked: number }; recordCount: number; datasetDraft: { dataset_id: string; tenant_id: string; project_id: string; schema_version: string; artifact_refs: string[]; description: string; record_count: number; created_by: string; }; } export class SyntheticDataFactory { constructor( private readonly generation: GenerationPort, private readonly privacy: PrivacyCheckPort, ) {} /** Kör hela kedjan; nekar publish om privacy/quality står sig ej. */ async run(input: { sourceDatasetId: string; sourceVersion: number; tenantId: string; projectId: string; sourceRows: SourceRow[]; evalRowSignatures?: readonly string[]; config: GenerationConfig; createdBy: string; }): Promise { const stages: PipelineStageReport[] = []; const cfg = input.config; // 1. sampling const sampled = input.sourceRows.slice(0, Math.max(1, Math.ceil(input.sourceRows.length / 2))); stages.push({ stage: 'sampled', inputCount: input.sourceRows.length, outputCount: sampled.length }); // 2. syntes (INF0) const { rows } = await this.generation.generate({ sourceRows: sampled, count: cfg.count, policyRef: cfg.generationPolicyRef, }); stages.push({ stage: 'synthesized', inputCount: sampled.length, outputCount: rows.length }); // 3. PII/privacy — fail-closed FÖRE publish const privacyChecks = { checked: 0, blocked: 0 }; const privacyOk: GeneratedRow[] = []; for (const row of rows) { privacyChecks.checked += 1; const cls = await this.privacy.classifyRow(row.fields); if (PII_ORDER[cls] > PII_ORDER[cfg.maxAllowedPiiClass]) { privacyChecks.blocked += 1; continue; } privacyOk.push(row); } stages.push({ stage: 'privacy', inputCount: rows.length, outputCount: privacyOk.length }); // 4. quality-filter const qualityOk = privacyOk.filter((r) => r.qualityScore >= cfg.minQualityScore); stages.push({ stage: 'quality', inputCount: privacyOk.length, outputCount: qualityOk.length }); // 5. dedupe (sha256 av fält) const seen = new Set(); const deduped: GeneratedRow[] = []; for (const r of qualityOk) { const h = createHash('sha256').update(JSON.stringify(r.fields)).digest('hex'); if (seen.has(h)) continue; seen.add(h); deduped.push({ ...r, provenance: { ...r.provenance, sha256: h } }); } stages.push({ stage: 'dedupe', inputCount: qualityOk.length, outputCount: deduped.length }); // 6. contamination — rader identiska med val/test-eval-signaturer raderas const evalSigs = new Set(input.evalRowSignatures ?? []); const clean = deduped.filter((r) => !evalSigs.has(r.provenance.sha256)); stages.push({ stage: 'contamination', inputCount: deduped.length, outputCount: clean.length }); // 7. train/val/test-split (deterministiskt via hash) const split = cfg.split ?? { train: 0.7, val: 0.15, test: 0.15 }; const outRows: SyntheticDatasetVersionDraft['rows'] = clean.map((r) => { const frac = parseInt(r.provenance.sha256.slice(0, 8), 16) / 0xffffffff; const s: 'train' | 'val' | 'test' = frac < split.train ? 'train' : frac < split.train + split.val ? 'val' : 'test'; return { split: s, provenance: r.provenance, fields: r.fields }; }); stages.push({ stage: 'split', inputCount: clean.length, outputCount: outRows.length }); const datasetId = `ds-synthetic-${input.sourceDatasetId}`; return { sourceDatasetId: input.sourceDatasetId, sourceVersion: input.sourceVersion, tenantId: input.tenantId, projectId: input.projectId, schemaVersion: 'synthetic/v1', description: `Synthetic Data Factory: ${input.sourceDatasetId}@${input.sourceVersion} (policy ${cfg.generationPolicyRef}, modell ${cfg.modelVersion})`, rows: outRows, stages, privacyChecks, recordCount: outRows.length, datasetDraft: { dataset_id: datasetId, tenant_id: input.tenantId, project_id: input.projectId, schema_version: 'synthetic/v1', artifact_refs: [`st0re://synthetic/${input.sourceDatasetId}/v${input.sourceVersion}`], description: `Synthetic (källa ${input.sourceDatasetId}@${input.sourceVersion}, policy ${cfg.generationPolicyRef})`, record_count: outRows.length, created_by: input.createdBy, }, }; } }