masterplan-lock / check (push) Successful in 5s
masterplan-lock / check (pull_request) Successful in 4s
CI (SIAX Cloud) / security (push) Successful in 18s
Security Scan (SIAX Cloud) / security (push) Successful in 18s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 14s
CI (SIAX Cloud) / quality (push) Successful in 38s
CI (SIAX Cloud) / quality (pull_request) Successful in 35s
CI (SIAX Cloud) / integration (pull_request) Successful in 51s
CI (SIAX Cloud) / security (pull_request) Successful in 1m18s
CI (SIAX Cloud) / integration (push) Failing after 1m30s
207 lines
7.3 KiB
TypeScript
207 lines
7.3 KiB
TypeScript
/**
|
||
* `0pt/synthetic/factory.ts` — X-737: Synthetic Data Factory.
|
||
*
|
||
* Ägande enligt kontraktet: 0pt äger dataset-truth + transformationer;
|
||
* INF0 äger generering (injectas via port). Pipeline:
|
||
*
|
||
* source DatasetVersion
|
||
* → sampling
|
||
* → synthesis (INF0 GenerationPort)
|
||
* → PII/privacy-check (fail-closed FÖRE publish)
|
||
* → quality-filter
|
||
* → dedupe
|
||
* → contamination-detection (rader = identiska med val/test-kandidater)
|
||
* → train/val/test-split
|
||
* → ny DatasetVersion (0pt-repot)
|
||
*
|
||
* PROVENANS PER RAD (acceptans): varje syntetisk rad bär generation-policy-ref
|
||
* + model-version + source-ref. Privacy-check blockerar publish.
|
||
*/
|
||
|
||
import { createHash } from 'node:crypto';
|
||
|
||
export type PiiClass = 'none' | 'low' | 'medium' | 'high';
|
||
|
||
export interface SourceRow {
|
||
id: string;
|
||
fields: Record<string, unknown>;
|
||
}
|
||
|
||
export interface GenerationConfig {
|
||
/** Policy-ref för genereringen (CL0UD/0pt-policy — spärras per rad). */
|
||
generationPolicyRef: string;
|
||
/** INF0-modellversion som genererade (provener per rad). */
|
||
modelVersion: string;
|
||
/** Antal rader som ska syntetiseras. */
|
||
count: number;
|
||
/** Högst tillåten PII-klass i publicerad data (över → NEKAS). */
|
||
maxAllowedPiiClass: PiiClass;
|
||
/** Quality-tröskel per rad (0–1). */
|
||
minQualityScore: number;
|
||
/** Train/val/test-fördelning (default 70/15/15). */
|
||
split?: { train: number; val: number; test: number };
|
||
}
|
||
|
||
export interface GeneratedRow {
|
||
fields: Record<string, unknown>;
|
||
/** Kvalitetspoäng från genereringen (0–1). */
|
||
qualityScore: number;
|
||
provenance: SyntheticRowProvenance;
|
||
}
|
||
|
||
export interface SyntheticRowProvenance {
|
||
rowId: string;
|
||
generationPolicyRef: string;
|
||
modelVersion: string;
|
||
sourceDatasetRef: string;
|
||
sourceRowIds: string[];
|
||
sha256: string;
|
||
}
|
||
|
||
/** INF0:s genereringsport (0pt anropar — INF0 äger modellen). */
|
||
export interface GenerationPort {
|
||
generate(input: {
|
||
sourceRows: SourceRow[];
|
||
count: number;
|
||
policyRef: string;
|
||
}): Promise<{ rows: GeneratedRow[] }>;
|
||
}
|
||
|
||
/** Privacy-port (X-728-presidio i INF0/integrations): klassificera rader. */
|
||
export interface PrivacyCheckPort {
|
||
classifyRow(fields: Record<string, unknown>): Promise<PiiClass>;
|
||
}
|
||
|
||
const PII_ORDER: Record<PiiClass, number> = { none: 0, low: 1, medium: 2, high: 3 };
|
||
|
||
export interface PipelineStageReport {
|
||
stage: 'sampled' | 'synthesized' | 'privacy' | 'quality' | 'dedupe' | 'contamination' | 'split';
|
||
inputCount: number;
|
||
outputCount: number;
|
||
}
|
||
|
||
export interface SyntheticDatasetVersionDraft {
|
||
sourceDatasetId: string;
|
||
sourceVersion: number;
|
||
tenantId: string;
|
||
projectId: string;
|
||
schemaVersion: string;
|
||
description: string;
|
||
rows: Array<{ split: 'train' | 'val' | 'test'; provenance: SyntheticRowProvenance; fields: Record<string, unknown> }>;
|
||
stages: PipelineStageReport[];
|
||
/** Privacy-checks före publish — bevis för acceptansen. */
|
||
privacyChecks: { checked: number; blocked: number };
|
||
recordCount: number;
|
||
datasetDraft: {
|
||
dataset_id: string;
|
||
tenant_id: string;
|
||
project_id: string;
|
||
schema_version: string;
|
||
artifact_refs: string[];
|
||
description: string;
|
||
record_count: number;
|
||
created_by: string;
|
||
};
|
||
}
|
||
|
||
export class SyntheticDataFactory {
|
||
constructor(
|
||
private readonly generation: GenerationPort,
|
||
private readonly privacy: PrivacyCheckPort,
|
||
) {}
|
||
|
||
/** Kör hela kedjan; nekar publish om privacy/quality står sig ej. */
|
||
async run(input: {
|
||
sourceDatasetId: string;
|
||
sourceVersion: number;
|
||
tenantId: string;
|
||
projectId: string;
|
||
sourceRows: SourceRow[];
|
||
evalRowSignatures?: readonly string[];
|
||
config: GenerationConfig;
|
||
createdBy: string;
|
||
}): Promise<SyntheticDatasetVersionDraft> {
|
||
const stages: PipelineStageReport[] = [];
|
||
const cfg = input.config;
|
||
|
||
// 1. sampling
|
||
const sampled = input.sourceRows.slice(0, Math.max(1, Math.ceil(input.sourceRows.length / 2)));
|
||
stages.push({ stage: 'sampled', inputCount: input.sourceRows.length, outputCount: sampled.length });
|
||
|
||
// 2. syntes (INF0)
|
||
const { rows } = await this.generation.generate({
|
||
sourceRows: sampled,
|
||
count: cfg.count,
|
||
policyRef: cfg.generationPolicyRef,
|
||
});
|
||
stages.push({ stage: 'synthesized', inputCount: sampled.length, outputCount: rows.length });
|
||
|
||
// 3. PII/privacy — fail-closed FÖRE publish
|
||
const privacyChecks = { checked: 0, blocked: 0 };
|
||
const privacyOk: GeneratedRow[] = [];
|
||
for (const row of rows) {
|
||
privacyChecks.checked += 1;
|
||
const cls = await this.privacy.classifyRow(row.fields);
|
||
if (PII_ORDER[cls] > PII_ORDER[cfg.maxAllowedPiiClass]) {
|
||
privacyChecks.blocked += 1;
|
||
continue;
|
||
}
|
||
privacyOk.push(row);
|
||
}
|
||
stages.push({ stage: 'privacy', inputCount: rows.length, outputCount: privacyOk.length });
|
||
|
||
// 4. quality-filter
|
||
const qualityOk = privacyOk.filter((r) => r.qualityScore >= cfg.minQualityScore);
|
||
stages.push({ stage: 'quality', inputCount: privacyOk.length, outputCount: qualityOk.length });
|
||
|
||
// 5. dedupe (sha256 av fält)
|
||
const seen = new Set<string>();
|
||
const deduped: GeneratedRow[] = [];
|
||
for (const r of qualityOk) {
|
||
const h = createHash('sha256').update(JSON.stringify(r.fields)).digest('hex');
|
||
if (seen.has(h)) continue;
|
||
seen.add(h);
|
||
deduped.push({ ...r, provenance: { ...r.provenance, sha256: h } });
|
||
}
|
||
stages.push({ stage: 'dedupe', inputCount: qualityOk.length, outputCount: deduped.length });
|
||
|
||
// 6. contamination — rader identiska med val/test-eval-signaturer raderas
|
||
const evalSigs = new Set(input.evalRowSignatures ?? []);
|
||
const clean = deduped.filter((r) => !evalSigs.has(r.provenance.sha256));
|
||
stages.push({ stage: 'contamination', inputCount: deduped.length, outputCount: clean.length });
|
||
|
||
// 7. train/val/test-split (deterministiskt via hash)
|
||
const split = cfg.split ?? { train: 0.7, val: 0.15, test: 0.15 };
|
||
const outRows: SyntheticDatasetVersionDraft['rows'] = clean.map((r) => {
|
||
const frac = parseInt(r.provenance.sha256.slice(0, 8), 16) / 0xffffffff;
|
||
const s: 'train' | 'val' | 'test' = frac < split.train ? 'train' : frac < split.train + split.val ? 'val' : 'test';
|
||
return { split: s, provenance: r.provenance, fields: r.fields };
|
||
});
|
||
stages.push({ stage: 'split', inputCount: clean.length, outputCount: outRows.length });
|
||
|
||
const datasetId = `ds-synthetic-${input.sourceDatasetId}`;
|
||
return {
|
||
sourceDatasetId: input.sourceDatasetId,
|
||
sourceVersion: input.sourceVersion,
|
||
tenantId: input.tenantId,
|
||
projectId: input.projectId,
|
||
schemaVersion: 'synthetic/v1',
|
||
description: `Synthetic Data Factory: ${input.sourceDatasetId}@${input.sourceVersion} (policy ${cfg.generationPolicyRef}, modell ${cfg.modelVersion})`,
|
||
rows: outRows,
|
||
stages,
|
||
privacyChecks,
|
||
recordCount: outRows.length,
|
||
datasetDraft: {
|
||
dataset_id: datasetId,
|
||
tenant_id: input.tenantId,
|
||
project_id: input.projectId,
|
||
schema_version: 'synthetic/v1',
|
||
artifact_refs: [`st0re://synthetic/${input.sourceDatasetId}/v${input.sourceVersion}`],
|
||
description: `Synthetic (källa ${input.sourceDatasetId}@${input.sourceVersion}, policy ${cfg.generationPolicyRef})`,
|
||
record_count: outRows.length,
|
||
created_by: input.createdBy,
|
||
},
|
||
};
|
||
}
|
||
}
|