Merge pull request 'feat(X-737): Synthetic Data Factory (0pt dataset-truth + INF0-generering)' (#77) from feat/x737-synthetic-factory into main
masterplan-lock / check (push) Successful in 10s
CI (SIAX Cloud) / security (push) Successful in 15s
Security Scan (SIAX Cloud) / security (push) Successful in 16s
CI (SIAX Cloud) / integration (push) Successful in 26s
CI (SIAX Cloud) / quality (push) Successful in 37s
Release chain (G17, ADR-W16-003) / Build, push, G17 (SBOM/scan/attest/verify/evidence) (push) Failing after 1m10s
Deploy to Coolify / deploy (push) Failing after 10m36s

This commit was merged in pull request #77.
This commit is contained in:
2026-09-21 11:14:41 +00:00
2 changed files with 363 additions and 0 deletions
+206
View File
@@ -0,0 +1,206 @@
/**
* `0pt/synthetic/factory.ts` — X-737: Synthetic Data Factory.
*
* Ägande enligt kontraktet: 0pt äger dataset-truth + transformationer;
* INF0 äger generering (injectas via port). Pipeline:
*
* source DatasetVersion
* → sampling
* → synthesis (INF0 GenerationPort)
* → PII/privacy-check (fail-closed FÖRE publish)
* → quality-filter
* → dedupe
* → contamination-detection (rader = identiska med val/test-kandidater)
* → train/val/test-split
* → ny DatasetVersion (0pt-repot)
*
* PROVENANS PER RAD (acceptans): varje syntetisk rad bär generation-policy-ref
* + model-version + source-ref. Privacy-check blockerar publish.
*/
import { createHash } from 'node:crypto';
export type PiiClass = 'none' | 'low' | 'medium' | 'high';
export interface SourceRow {
id: string;
fields: Record<string, unknown>;
}
export interface GenerationConfig {
/** Policy-ref för genereringen (CL0UD/0pt-policy — spärras per rad). */
generationPolicyRef: string;
/** INF0-modellversion som genererade (provener per rad). */
modelVersion: string;
/** Antal rader som ska syntetiseras. */
count: number;
/** Högst tillåten PII-klass i publicerad data (över → NEKAS). */
maxAllowedPiiClass: PiiClass;
/** Quality-tröskel per rad (0–1). */
minQualityScore: number;
/** Train/val/test-fördelning (default 70/15/15). */
split?: { train: number; val: number; test: number };
}
export interface GeneratedRow {
fields: Record<string, unknown>;
/** Kvalitetspoäng från genereringen (0–1). */
qualityScore: number;
provenance: SyntheticRowProvenance;
}
export interface SyntheticRowProvenance {
rowId: string;
generationPolicyRef: string;
modelVersion: string;
sourceDatasetRef: string;
sourceRowIds: string[];
sha256: string;
}
/** INF0:s genereringsport (0pt anropar — INF0 äger modellen). */
export interface GenerationPort {
generate(input: {
sourceRows: SourceRow[];
count: number;
policyRef: string;
}): Promise<{ rows: GeneratedRow[] }>;
}
/** Privacy-port (X-728-presidio i INF0/integrations): klassificera rader. */
export interface PrivacyCheckPort {
classifyRow(fields: Record<string, unknown>): Promise<PiiClass>;
}
const PII_ORDER: Record<PiiClass, number> = { none: 0, low: 1, medium: 2, high: 3 };
export interface PipelineStageReport {
stage: 'sampled' | 'synthesized' | 'privacy' | 'quality' | 'dedupe' | 'contamination' | 'split';
inputCount: number;
outputCount: number;
}
export interface SyntheticDatasetVersionDraft {
sourceDatasetId: string;
sourceVersion: number;
tenantId: string;
projectId: string;
schemaVersion: string;
description: string;
rows: Array<{ split: 'train' | 'val' | 'test'; provenance: SyntheticRowProvenance; fields: Record<string, unknown> }>;
stages: PipelineStageReport[];
/** Privacy-checks före publish — bevis för acceptansen. */
privacyChecks: { checked: number; blocked: number };
recordCount: number;
datasetDraft: {
dataset_id: string;
tenant_id: string;
project_id: string;
schema_version: string;
artifact_refs: string[];
description: string;
record_count: number;
created_by: string;
};
}
export class SyntheticDataFactory {
constructor(
private readonly generation: GenerationPort,
private readonly privacy: PrivacyCheckPort,
) {}
/** Kör hela kedjan; nekar publish om privacy/quality står sig ej. */
async run(input: {
sourceDatasetId: string;
sourceVersion: number;
tenantId: string;
projectId: string;
sourceRows: SourceRow[];
evalRowSignatures?: readonly string[];
config: GenerationConfig;
createdBy: string;
}): Promise<SyntheticDatasetVersionDraft> {
const stages: PipelineStageReport[] = [];
const cfg = input.config;
// 1. sampling
const sampled = input.sourceRows.slice(0, Math.max(1, Math.ceil(input.sourceRows.length / 2)));
stages.push({ stage: 'sampled', inputCount: input.sourceRows.length, outputCount: sampled.length });
// 2. syntes (INF0)
const { rows } = await this.generation.generate({
sourceRows: sampled,
count: cfg.count,
policyRef: cfg.generationPolicyRef,
});
stages.push({ stage: 'synthesized', inputCount: sampled.length, outputCount: rows.length });
// 3. PII/privacy — fail-closed FÖRE publish
const privacyChecks = { checked: 0, blocked: 0 };
const privacyOk: GeneratedRow[] = [];
for (const row of rows) {
privacyChecks.checked += 1;
const cls = await this.privacy.classifyRow(row.fields);
if (PII_ORDER[cls] > PII_ORDER[cfg.maxAllowedPiiClass]) {
privacyChecks.blocked += 1;
continue;
}
privacyOk.push(row);
}
stages.push({ stage: 'privacy', inputCount: rows.length, outputCount: privacyOk.length });
// 4. quality-filter
const qualityOk = privacyOk.filter((r) => r.qualityScore >= cfg.minQualityScore);
stages.push({ stage: 'quality', inputCount: privacyOk.length, outputCount: qualityOk.length });
// 5. dedupe (sha256 av fält)
const seen = new Set<string>();
const deduped: GeneratedRow[] = [];
for (const r of qualityOk) {
const h = createHash('sha256').update(JSON.stringify(r.fields)).digest('hex');
if (seen.has(h)) continue;
seen.add(h);
deduped.push({ ...r, provenance: { ...r.provenance, sha256: h } });
}
stages.push({ stage: 'dedupe', inputCount: qualityOk.length, outputCount: deduped.length });
// 6. contamination — rader identiska med val/test-eval-signaturer raderas
const evalSigs = new Set(input.evalRowSignatures ?? []);
const clean = deduped.filter((r) => !evalSigs.has(r.provenance.sha256));
stages.push({ stage: 'contamination', inputCount: deduped.length, outputCount: clean.length });
// 7. train/val/test-split (deterministiskt via hash)
const split = cfg.split ?? { train: 0.7, val: 0.15, test: 0.15 };
const outRows: SyntheticDatasetVersionDraft['rows'] = clean.map((r) => {
const frac = parseInt(r.provenance.sha256.slice(0, 8), 16) / 0xffffffff;
const s: 'train' | 'val' | 'test' = frac < split.train ? 'train' : frac < split.train + split.val ? 'val' : 'test';
return { split: s, provenance: r.provenance, fields: r.fields };
});
stages.push({ stage: 'split', inputCount: clean.length, outputCount: outRows.length });
const datasetId = `ds-synthetic-${input.sourceDatasetId}`;
return {
sourceDatasetId: input.sourceDatasetId,
sourceVersion: input.sourceVersion,
tenantId: input.tenantId,
projectId: input.projectId,
schemaVersion: 'synthetic/v1',
description: `Synthetic Data Factory: ${input.sourceDatasetId}@${input.sourceVersion} (policy ${cfg.generationPolicyRef}, modell ${cfg.modelVersion})`,
rows: outRows,
stages,
privacyChecks,
recordCount: outRows.length,
datasetDraft: {
dataset_id: datasetId,
tenant_id: input.tenantId,
project_id: input.projectId,
schema_version: 'synthetic/v1',
artifact_refs: [`st0re://synthetic/${input.sourceDatasetId}/v${input.sourceVersion}`],
description: `Synthetic (källa ${input.sourceDatasetId}@${input.sourceVersion}, policy ${cfg.generationPolicyRef})`,
record_count: outRows.length,
created_by: input.createdBy,
},
};
}
}