refactor: persist dataset versions and make lineage retry-stable

This commit is contained in:
2026-09-07 18:03:21 +02:00
parent 3ccdcf0cf6
commit 3c5850fbe5
+69 -113
View File
@@ -1,96 +1,62 @@
/**
* src/services/dataset-version-service.ts
* ========================================
* P0 Dataset capability extensions (Master-DoD §10 Dataset:
* dataset.create/version/query/export) layered ON TOP of DatasetService (the
* owner of 0pt.dataset.schema.v1 registry entries — deliberately untouched:
* riv inte fungerande kod).
* Dataset version/query/export capability.
*
* dataset.version -> createVersion: an immutable, monotonically numbered
* version whose artifacts are CANONICAL ST0RE REFS
* (`st0re:artifact:art_x` — Master-DoD §5); 0pt never
* stores bytes here, only references.
* dataset.query -> listVersions/getVersion (tenant-gated).
* dataset.export -> exportDataset: registers the export artifact in ST0RE
* (real client), records AUD0 evidence + OpenLineage,
* returns the canonical `st0re:artifact:...` ref.
*
* Idempotency (§53): createVersion accepts an idempotency_key; the same key
* returns the SAME version instead of minting a duplicate.
* Dataset metadata is canonical in DatasetService; immutable version metadata
* is persisted through DatasetVersionRepository; bytes remain canonical ST0RE
* artifacts. AUD0/OpenLineage record the evidence chain.
*/
import crypto from "crypto";
import { SiaxError } from "../platform/errors";
import { formatResourceRef, isResourceRef } from "../platform/resource-ref";
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import { createSt0reClient, type St0reClient } from "../clients/st0re-client";
import { LineageEmitter } from "../lineage/lineage-emitter";
import {
createDatasetVersionRepository,
type DatasetVersionRepository,
} from "../repositories/dataset-version-repository";
import { SiaxError } from "../platform/errors";
import { formatResourceRef, isResourceRef } from "../platform/resource-ref";
import type {
CreateDatasetVersionInput,
DatasetExport,
DatasetVersion,
} from "../types/dataset-version";
import { DatasetService } from "./dataset-service";
export interface DatasetVersion {
dataset_id: string;
version: number;
tenant_id: string;
project_id: string;
/** Canonical ST0RE artifact refs (st0re:artifact:...) making up this version. */
artifact_refs: string[];
/** Schema version of the dataset content at this version. */
schema_version: string;
/** Row/record count when known. */
record_count?: number;
description?: string;
created_at: string;
created_by: string;
correlation_id?: string;
}
export type { CreateDatasetVersionInput, DatasetExport, DatasetVersion } from "../types/dataset-version";
export interface CreateDatasetVersionInput {
dataset_id: string;
tenant_id: string;
project_id: string;
artifact_refs: string[];
schema_version: string;
record_count?: number;
description?: string;
created_by: string;
correlation_id?: string;
idempotency_key?: string;
}
export interface DatasetExport {
export_id: string;
dataset_id: string;
version: number;
tenant_id: string;
/** Canonical ref of the registered export artifact. */
artifact_ref: string;
format: "parquet" | "csv" | "json";
created_at: string;
export interface DatasetVersionServiceOptions {
aud0BaseUrl?: string;
st0reBaseUrl?: string;
databaseUrl?: string;
datasetService?: DatasetService;
repository?: DatasetVersionRepository;
}
export class DatasetVersionService {
private readonly versions = new Map<string, DatasetVersion[]>();
private readonly idempotency = new Map<string, DatasetVersion>();
private readonly repo: DatasetVersionRepository;
private readonly aud0: Aud0Client;
private readonly st0re: St0reClient;
private readonly lineage: LineageEmitter;
readonly datasets: DatasetService;
constructor(opts?: { aud0BaseUrl?: string; st0reBaseUrl?: string; datasetService?: DatasetService }) {
constructor(opts?: DatasetVersionServiceOptions) {
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.st0re = createSt0reClient(opts?.st0reBaseUrl);
this.lineage = new LineageEmitter(this.aud0);
this.datasets = opts?.datasetService ?? new DatasetService(opts?.aud0BaseUrl);
this.datasets = opts?.datasetService ?? new DatasetService({
aud0BaseUrl: opts?.aud0BaseUrl,
databaseUrl: opts?.databaseUrl,
});
this.repo = opts?.repository ?? createDatasetVersionRepository(opts?.databaseUrl);
}
// -----------------------------------------------------------------------
// dataset.version
// -----------------------------------------------------------------------
async createVersion(input: CreateDatasetVersionInput): Promise<DatasetVersion> {
await this.assertDatasetTenant(input.dataset_id, input.tenant_id);
if (input.idempotency_key) {
const existing = this.idempotency.get(`${input.tenant_id}:${input.idempotency_key}`);
if (existing) return existing;
if (!input.project_id?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "project_id is required");
if (!input.schema_version?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "schema_version is required");
if (!input.created_by?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "created_by is required");
if (input.record_count !== undefined && (!Number.isSafeInteger(input.record_count) || input.record_count < 0)) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "record_count must be a non-negative safe integer");
}
if (input.artifact_refs.length === 0) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "a dataset version needs at least one artifact ref");
@@ -100,37 +66,19 @@ export class DatasetVersionService {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.dataset",
`artifact ref "${ref}" is not a canonical st0re:artifact:... reference (Master-DoD §5)`
`artifact ref "${ref}" is not a canonical st0re:artifact:... reference (Master-DoD §5)`,
);
}
}
const list = this.versions.get(input.dataset_id) ?? [];
const version: DatasetVersion = {
dataset_id: input.dataset_id,
version: (list[list.length - 1]?.version ?? 0) + 1,
tenant_id: input.tenant_id,
project_id: input.project_id,
artifact_refs: [...input.artifact_refs],
schema_version: input.schema_version,
record_count: input.record_count,
description: input.description,
created_at: new Date().toISOString(),
created_by: input.created_by,
correlation_id: input.correlation_id,
};
list.push(version);
this.versions.set(input.dataset_id, list);
if (input.idempotency_key) {
this.idempotency.set(`${input.tenant_id}:${input.idempotency_key}`, version);
}
const version = await this.repo.create(input);
const versionRef = formatResourceRef({
product: "0pt",
resourceType: "dataset",
resourceId: input.dataset_id,
version: String(version.version),
});
await this.aud0.ingestEvent({
event_id: `evt_0pt_dataset_version_${input.dataset_id}_v${version.version}`,
event_type: "0pt.dataset_version_created",
@@ -145,10 +93,11 @@ export class DatasetVersionService {
result: "success",
input_refs: version.artifact_refs,
output_refs: [versionRef],
payload: { schema_version: input.schema_version, record_count: input.record_count },
payload: { schema_version: version.schema_version, record_count: version.record_count },
created_at: version.created_at,
correlation_id: input.correlation_id,
idempotency_key: input.idempotency_key,
occurred_at: version.created_at,
correlation_id: version.correlation_id,
idempotency_key: `dataset-version:${input.tenant_id}:${input.dataset_id}:v${version.version}`,
});
await this.lineage.emit({
tenant_id: input.tenant_id,
@@ -156,41 +105,34 @@ export class DatasetVersionService {
capability_id: "0pt.dataset.version.v1",
run_id: `dataset-version:${input.dataset_id}:v${version.version}`,
event_type: "COMPLETE",
event_time: version.created_at,
inputs: version.artifact_refs.map((ref) => ({ name: ref, ref })),
outputs: [{ name: `dataset/${input.dataset_id}`, ref: versionRef }],
correlation_id: input.correlation_id,
correlation_id: version.correlation_id,
});
return version;
}
// -----------------------------------------------------------------------
// dataset.query
// -----------------------------------------------------------------------
async listVersions(datasetId: string, tenantId: string): Promise<DatasetVersion[]> {
await this.assertDatasetTenant(datasetId, tenantId);
return [...(this.versions.get(datasetId) ?? [])];
return this.repo.list(datasetId, tenantId);
}
async getVersion(datasetId: string, tenantId: string, version: number): Promise<DatasetVersion> {
const list = await this.listVersions(datasetId, tenantId);
const found = list.find((v) => v.version === version);
if (!found) {
throw new SiaxError("NOT_FOUND", "0pt.dataset", `dataset ${datasetId} has no version ${version}`);
await this.assertDatasetTenant(datasetId, tenantId);
if (!Number.isSafeInteger(version) || version < 1) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "version must be a positive integer");
}
return found;
return this.repo.get(datasetId, tenantId, version);
}
// -----------------------------------------------------------------------
// dataset.export
// -----------------------------------------------------------------------
async exportDataset(input: {
dataset_id: string;
tenant_id: string;
project_id: string;
version: number;
format: DatasetExport["format"];
/** Where the export bytes were written (by the exporting job). */
object_location: { provider: string; bucket: string; key: string; region?: string };
size_bytes: number;
checksum_sha256: string;
@@ -198,7 +140,18 @@ export class DatasetVersionService {
correlation_id?: string;
}): Promise<DatasetExport> {
const version = await this.getVersion(input.dataset_id, input.tenant_id, input.version);
const exportId = `dsexp_${crypto.randomUUID()}`;
if (!(["parquet", "csv", "json"] as const).includes(input.format)) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "unsupported export format");
}
if (!Number.isSafeInteger(input.size_bytes) || input.size_bytes < 0) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "size_bytes must be a non-negative safe integer");
}
if (!/^[a-fA-F0-9]{64}$/.test(input.checksum_sha256)) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "checksum_sha256 must be a 64-character hex digest");
}
const exportKey = `dsexp:${input.tenant_id}:${input.dataset_id}:v${input.version}:${input.format}:${input.checksum_sha256.toLowerCase()}`;
const exportId = `dsexp_${crypto.createHash("sha256").update(exportKey).digest("hex").slice(0, 32)}`;
const artifact = await this.st0re.createArtifact({
tenant_id: input.tenant_id,
@@ -214,14 +167,14 @@ export class DatasetVersionService {
? "text/csv"
: "application/json",
size_bytes: input.size_bytes,
checksum_sha256: input.checksum_sha256,
checksum_sha256: input.checksum_sha256.toLowerCase(),
pii_level: "none",
retention_policy: "standard",
data_residency: "eu",
risk_level: "L2",
correlation_id: input.correlation_id,
created_by: input.actor_id,
idempotency_key: `dsexp:${input.dataset_id}:v${input.version}:${input.format}:${input.checksum_sha256}`,
idempotency_key: exportKey,
});
const artifactRef = formatResourceRef({
@@ -229,6 +182,7 @@ export class DatasetVersionService {
resourceType: "artifact",
resourceId: artifact.artifact_id,
});
const createdAt = artifact.created_at ?? new Date().toISOString();
const record: DatasetExport = {
export_id: exportId,
dataset_id: input.dataset_id,
@@ -236,7 +190,7 @@ export class DatasetVersionService {
tenant_id: input.tenant_id,
artifact_ref: artifactRef,
format: input.format,
created_at: new Date().toISOString(),
created_at: createdAt,
};
const versionRef = formatResourceRef({
@@ -260,8 +214,10 @@ export class DatasetVersionService {
input_refs: [versionRef],
output_refs: [artifactRef],
payload: { format: input.format, size_bytes: input.size_bytes },
created_at: record.created_at,
created_at: createdAt,
occurred_at: createdAt,
correlation_id: input.correlation_id,
idempotency_key: exportKey,
});
await this.lineage.emit({
tenant_id: input.tenant_id,
@@ -269,6 +225,7 @@ export class DatasetVersionService {
capability_id: "0pt.dataset.export.v1",
run_id: `dataset-export:${exportId}`,
event_type: "COMPLETE",
event_time: createdAt,
inputs: [{ name: `dataset/${input.dataset_id}`, ref: versionRef }],
outputs: [{ name: artifactRef, ref: artifactRef }],
correlation_id: input.correlation_id,
@@ -277,9 +234,8 @@ export class DatasetVersionService {
return record;
}
// -----------------------------------------------------------------------
private async assertDatasetTenant(datasetId: string, tenantId: string): Promise<void> {
if (!tenantId) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "tenant_id is required");
if (!tenantId?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "tenant_id is required");
let dataset;
try {
dataset = await this.datasets.getDataset(datasetId);