Files
0pt/src/synthetic/factory.ts
T
admin 8411f9759e
masterplan-lock / check (push) Successful in 5s
masterplan-lock / check (pull_request) Successful in 4s
CI (SIAX Cloud) / security (push) Successful in 18s
Security Scan (SIAX Cloud) / security (push) Successful in 18s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 14s
CI (SIAX Cloud) / quality (push) Successful in 38s
CI (SIAX Cloud) / quality (pull_request) Successful in 35s
CI (SIAX Cloud) / integration (pull_request) Successful in 51s
CI (SIAX Cloud) / security (pull_request) Successful in 1m18s
CI (SIAX Cloud) / integration (push) Failing after 1m30s
feat(X-737): Synthetic Data Factory — sampling→synthesis (INF0-port)→PII/privacy (fail-closed före publish)→quality→dedupe→contamination→deterministisk train/val/test-split; provenance per rad (policy+model+source+sha256). 5/5 tester
2026-09-21 13:07:54 +02:00

207 lines
7.3 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* `0pt/synthetic/factory.ts` — X-737: Synthetic Data Factory.
*
* Ägande enligt kontraktet: 0pt äger dataset-truth + transformationer;
* INF0 äger generering (injectas via port). Pipeline:
*
* source DatasetVersion
* → sampling
* → synthesis (INF0 GenerationPort)
* → PII/privacy-check (fail-closed FÖRE publish)
* → quality-filter
* → dedupe
* → contamination-detection (rader = identiska med val/test-kandidater)
* → train/val/test-split
* → ny DatasetVersion (0pt-repot)
*
* PROVENANS PER RAD (acceptans): varje syntetisk rad bär generation-policy-ref
* + model-version + source-ref. Privacy-check blockerar publish.
*/
import { createHash } from 'node:crypto';
export type PiiClass = 'none' | 'low' | 'medium' | 'high';
export interface SourceRow {
id: string;
fields: Record<string, unknown>;
}
export interface GenerationConfig {
/** Policy-ref för genereringen (CL0UD/0pt-policy — spärras per rad). */
generationPolicyRef: string;
/** INF0-modellversion som genererade (provener per rad). */
modelVersion: string;
/** Antal rader som ska syntetiseras. */
count: number;
/** Högst tillåten PII-klass i publicerad data (över → NEKAS). */
maxAllowedPiiClass: PiiClass;
/** Quality-tröskel per rad (0–1). */
minQualityScore: number;
/** Train/val/test-fördelning (default 70/15/15). */
split?: { train: number; val: number; test: number };
}
export interface GeneratedRow {
fields: Record<string, unknown>;
/** Kvalitetspoäng från genereringen (0–1). */
qualityScore: number;
provenance: SyntheticRowProvenance;
}
export interface SyntheticRowProvenance {
rowId: string;
generationPolicyRef: string;
modelVersion: string;
sourceDatasetRef: string;
sourceRowIds: string[];
sha256: string;
}
/** INF0:s genereringsport (0pt anropar — INF0 äger modellen). */
export interface GenerationPort {
generate(input: {
sourceRows: SourceRow[];
count: number;
policyRef: string;
}): Promise<{ rows: GeneratedRow[] }>;
}
/** Privacy-port (X-728-presidio i INF0/integrations): klassificera rader. */
export interface PrivacyCheckPort {
classifyRow(fields: Record<string, unknown>): Promise<PiiClass>;
}
const PII_ORDER: Record<PiiClass, number> = { none: 0, low: 1, medium: 2, high: 3 };
export interface PipelineStageReport {
stage: 'sampled' | 'synthesized' | 'privacy' | 'quality' | 'dedupe' | 'contamination' | 'split';
inputCount: number;
outputCount: number;
}
export interface SyntheticDatasetVersionDraft {
sourceDatasetId: string;
sourceVersion: number;
tenantId: string;
projectId: string;
schemaVersion: string;
description: string;
rows: Array<{ split: 'train' | 'val' | 'test'; provenance: SyntheticRowProvenance; fields: Record<string, unknown> }>;
stages: PipelineStageReport[];
/** Privacy-checks före publish — bevis för acceptansen. */
privacyChecks: { checked: number; blocked: number };
recordCount: number;
datasetDraft: {
dataset_id: string;
tenant_id: string;
project_id: string;
schema_version: string;
artifact_refs: string[];
description: string;
record_count: number;
created_by: string;
};
}
export class SyntheticDataFactory {
constructor(
private readonly generation: GenerationPort,
private readonly privacy: PrivacyCheckPort,
) {}
/** Kör hela kedjan; nekar publish om privacy/quality står sig ej. */
async run(input: {
sourceDatasetId: string;
sourceVersion: number;
tenantId: string;
projectId: string;
sourceRows: SourceRow[];
evalRowSignatures?: readonly string[];
config: GenerationConfig;
createdBy: string;
}): Promise<SyntheticDatasetVersionDraft> {
const stages: PipelineStageReport[] = [];
const cfg = input.config;
// 1. sampling
const sampled = input.sourceRows.slice(0, Math.max(1, Math.ceil(input.sourceRows.length / 2)));
stages.push({ stage: 'sampled', inputCount: input.sourceRows.length, outputCount: sampled.length });
// 2. syntes (INF0)
const { rows } = await this.generation.generate({
sourceRows: sampled,
count: cfg.count,
policyRef: cfg.generationPolicyRef,
});
stages.push({ stage: 'synthesized', inputCount: sampled.length, outputCount: rows.length });
// 3. PII/privacy — fail-closed FÖRE publish
const privacyChecks = { checked: 0, blocked: 0 };
const privacyOk: GeneratedRow[] = [];
for (const row of rows) {
privacyChecks.checked += 1;
const cls = await this.privacy.classifyRow(row.fields);
if (PII_ORDER[cls] > PII_ORDER[cfg.maxAllowedPiiClass]) {
privacyChecks.blocked += 1;
continue;
}
privacyOk.push(row);
}
stages.push({ stage: 'privacy', inputCount: rows.length, outputCount: privacyOk.length });
// 4. quality-filter
const qualityOk = privacyOk.filter((r) => r.qualityScore >= cfg.minQualityScore);
stages.push({ stage: 'quality', inputCount: privacyOk.length, outputCount: qualityOk.length });
// 5. dedupe (sha256 av fält)
const seen = new Set<string>();
const deduped: GeneratedRow[] = [];
for (const r of qualityOk) {
const h = createHash('sha256').update(JSON.stringify(r.fields)).digest('hex');
if (seen.has(h)) continue;
seen.add(h);
deduped.push({ ...r, provenance: { ...r.provenance, sha256: h } });
}
stages.push({ stage: 'dedupe', inputCount: qualityOk.length, outputCount: deduped.length });
// 6. contamination — rader identiska med val/test-eval-signaturer raderas
const evalSigs = new Set(input.evalRowSignatures ?? []);
const clean = deduped.filter((r) => !evalSigs.has(r.provenance.sha256));
stages.push({ stage: 'contamination', inputCount: deduped.length, outputCount: clean.length });
// 7. train/val/test-split (deterministiskt via hash)
const split = cfg.split ?? { train: 0.7, val: 0.15, test: 0.15 };
const outRows: SyntheticDatasetVersionDraft['rows'] = clean.map((r) => {
const frac = parseInt(r.provenance.sha256.slice(0, 8), 16) / 0xffffffff;
const s: 'train' | 'val' | 'test' = frac < split.train ? 'train' : frac < split.train + split.val ? 'val' : 'test';
return { split: s, provenance: r.provenance, fields: r.fields };
});
stages.push({ stage: 'split', inputCount: clean.length, outputCount: outRows.length });
const datasetId = `ds-synthetic-${input.sourceDatasetId}`;
return {
sourceDatasetId: input.sourceDatasetId,
sourceVersion: input.sourceVersion,
tenantId: input.tenantId,
projectId: input.projectId,
schemaVersion: 'synthetic/v1',
description: `Synthetic Data Factory: ${input.sourceDatasetId}@${input.sourceVersion} (policy ${cfg.generationPolicyRef}, modell ${cfg.modelVersion})`,
rows: outRows,
stages,
privacyChecks,
recordCount: outRows.length,
datasetDraft: {
dataset_id: datasetId,
tenant_id: input.tenantId,
project_id: input.projectId,
schema_version: 'synthetic/v1',
artifact_refs: [`st0re://synthetic/${input.sourceDatasetId}/v${input.sourceVersion}`],
description: `Synthetic (källa ${input.sourceDatasetId}@${input.sourceVersion}, policy ${cfg.generationPolicyRef})`,
record_count: outRows.length,
created_by: input.createdBy,
},
};
}
}