refactor: persist dataset versions and make lineage retry-stable
This commit is contained in:
@@ -1,96 +1,62 @@
|
||||
/**
|
||||
* src/services/dataset-version-service.ts
|
||||
* ========================================
|
||||
* P0 Dataset capability extensions (Master-DoD §10 Dataset:
|
||||
* dataset.create/version/query/export) layered ON TOP of DatasetService (the
|
||||
* owner of 0pt.dataset.schema.v1 registry entries — deliberately untouched:
|
||||
* riv inte fungerande kod).
|
||||
* Dataset version/query/export capability.
|
||||
*
|
||||
* dataset.version -> createVersion: an immutable, monotonically numbered
|
||||
* version whose artifacts are CANONICAL ST0RE REFS
|
||||
* (`st0re:artifact:art_x` — Master-DoD §5); 0pt never
|
||||
* stores bytes here, only references.
|
||||
* dataset.query -> listVersions/getVersion (tenant-gated).
|
||||
* dataset.export -> exportDataset: registers the export artifact in ST0RE
|
||||
* (real client), records AUD0 evidence + OpenLineage,
|
||||
* returns the canonical `st0re:artifact:...` ref.
|
||||
*
|
||||
* Idempotency (§53): createVersion accepts an idempotency_key; the same key
|
||||
* returns the SAME version instead of minting a duplicate.
|
||||
* Dataset metadata is canonical in DatasetService; immutable version metadata
|
||||
* is persisted through DatasetVersionRepository; bytes remain canonical ST0RE
|
||||
* artifacts. AUD0/OpenLineage record the evidence chain.
|
||||
*/
|
||||
import crypto from "crypto";
|
||||
import { SiaxError } from "../platform/errors";
|
||||
import { formatResourceRef, isResourceRef } from "../platform/resource-ref";
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import { createSt0reClient, type St0reClient } from "../clients/st0re-client";
|
||||
import { LineageEmitter } from "../lineage/lineage-emitter";
|
||||
import {
|
||||
createDatasetVersionRepository,
|
||||
type DatasetVersionRepository,
|
||||
} from "../repositories/dataset-version-repository";
|
||||
import { SiaxError } from "../platform/errors";
|
||||
import { formatResourceRef, isResourceRef } from "../platform/resource-ref";
|
||||
import type {
|
||||
CreateDatasetVersionInput,
|
||||
DatasetExport,
|
||||
DatasetVersion,
|
||||
} from "../types/dataset-version";
|
||||
import { DatasetService } from "./dataset-service";
|
||||
|
||||
export interface DatasetVersion {
|
||||
dataset_id: string;
|
||||
version: number;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
/** Canonical ST0RE artifact refs (st0re:artifact:...) making up this version. */
|
||||
artifact_refs: string[];
|
||||
/** Schema version of the dataset content at this version. */
|
||||
schema_version: string;
|
||||
/** Row/record count when known. */
|
||||
record_count?: number;
|
||||
description?: string;
|
||||
created_at: string;
|
||||
created_by: string;
|
||||
correlation_id?: string;
|
||||
}
|
||||
export type { CreateDatasetVersionInput, DatasetExport, DatasetVersion } from "../types/dataset-version";
|
||||
|
||||
export interface CreateDatasetVersionInput {
|
||||
dataset_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
artifact_refs: string[];
|
||||
schema_version: string;
|
||||
record_count?: number;
|
||||
description?: string;
|
||||
created_by: string;
|
||||
correlation_id?: string;
|
||||
idempotency_key?: string;
|
||||
}
|
||||
|
||||
export interface DatasetExport {
|
||||
export_id: string;
|
||||
dataset_id: string;
|
||||
version: number;
|
||||
tenant_id: string;
|
||||
/** Canonical ref of the registered export artifact. */
|
||||
artifact_ref: string;
|
||||
format: "parquet" | "csv" | "json";
|
||||
created_at: string;
|
||||
export interface DatasetVersionServiceOptions {
|
||||
aud0BaseUrl?: string;
|
||||
st0reBaseUrl?: string;
|
||||
databaseUrl?: string;
|
||||
datasetService?: DatasetService;
|
||||
repository?: DatasetVersionRepository;
|
||||
}
|
||||
|
||||
export class DatasetVersionService {
|
||||
private readonly versions = new Map<string, DatasetVersion[]>();
|
||||
private readonly idempotency = new Map<string, DatasetVersion>();
|
||||
private readonly repo: DatasetVersionRepository;
|
||||
private readonly aud0: Aud0Client;
|
||||
private readonly st0re: St0reClient;
|
||||
private readonly lineage: LineageEmitter;
|
||||
readonly datasets: DatasetService;
|
||||
|
||||
constructor(opts?: { aud0BaseUrl?: string; st0reBaseUrl?: string; datasetService?: DatasetService }) {
|
||||
constructor(opts?: DatasetVersionServiceOptions) {
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
this.st0re = createSt0reClient(opts?.st0reBaseUrl);
|
||||
this.lineage = new LineageEmitter(this.aud0);
|
||||
this.datasets = opts?.datasetService ?? new DatasetService(opts?.aud0BaseUrl);
|
||||
this.datasets = opts?.datasetService ?? new DatasetService({
|
||||
aud0BaseUrl: opts?.aud0BaseUrl,
|
||||
databaseUrl: opts?.databaseUrl,
|
||||
});
|
||||
this.repo = opts?.repository ?? createDatasetVersionRepository(opts?.databaseUrl);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// dataset.version
|
||||
// -----------------------------------------------------------------------
|
||||
async createVersion(input: CreateDatasetVersionInput): Promise<DatasetVersion> {
|
||||
await this.assertDatasetTenant(input.dataset_id, input.tenant_id);
|
||||
|
||||
if (input.idempotency_key) {
|
||||
const existing = this.idempotency.get(`${input.tenant_id}:${input.idempotency_key}`);
|
||||
if (existing) return existing;
|
||||
if (!input.project_id?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "project_id is required");
|
||||
if (!input.schema_version?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "schema_version is required");
|
||||
if (!input.created_by?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "created_by is required");
|
||||
if (input.record_count !== undefined && (!Number.isSafeInteger(input.record_count) || input.record_count < 0)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "record_count must be a non-negative safe integer");
|
||||
}
|
||||
if (input.artifact_refs.length === 0) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "a dataset version needs at least one artifact ref");
|
||||
@@ -100,37 +66,19 @@ export class DatasetVersionService {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.dataset",
|
||||
`artifact ref "${ref}" is not a canonical st0re:artifact:... reference (Master-DoD §5)`
|
||||
`artifact ref "${ref}" is not a canonical st0re:artifact:... reference (Master-DoD §5)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const list = this.versions.get(input.dataset_id) ?? [];
|
||||
const version: DatasetVersion = {
|
||||
dataset_id: input.dataset_id,
|
||||
version: (list[list.length - 1]?.version ?? 0) + 1,
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
artifact_refs: [...input.artifact_refs],
|
||||
schema_version: input.schema_version,
|
||||
record_count: input.record_count,
|
||||
description: input.description,
|
||||
created_at: new Date().toISOString(),
|
||||
created_by: input.created_by,
|
||||
correlation_id: input.correlation_id,
|
||||
};
|
||||
list.push(version);
|
||||
this.versions.set(input.dataset_id, list);
|
||||
if (input.idempotency_key) {
|
||||
this.idempotency.set(`${input.tenant_id}:${input.idempotency_key}`, version);
|
||||
}
|
||||
|
||||
const version = await this.repo.create(input);
|
||||
const versionRef = formatResourceRef({
|
||||
product: "0pt",
|
||||
resourceType: "dataset",
|
||||
resourceId: input.dataset_id,
|
||||
version: String(version.version),
|
||||
});
|
||||
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_dataset_version_${input.dataset_id}_v${version.version}`,
|
||||
event_type: "0pt.dataset_version_created",
|
||||
@@ -145,10 +93,11 @@ export class DatasetVersionService {
|
||||
result: "success",
|
||||
input_refs: version.artifact_refs,
|
||||
output_refs: [versionRef],
|
||||
payload: { schema_version: input.schema_version, record_count: input.record_count },
|
||||
payload: { schema_version: version.schema_version, record_count: version.record_count },
|
||||
created_at: version.created_at,
|
||||
correlation_id: input.correlation_id,
|
||||
idempotency_key: input.idempotency_key,
|
||||
occurred_at: version.created_at,
|
||||
correlation_id: version.correlation_id,
|
||||
idempotency_key: `dataset-version:${input.tenant_id}:${input.dataset_id}:v${version.version}`,
|
||||
});
|
||||
await this.lineage.emit({
|
||||
tenant_id: input.tenant_id,
|
||||
@@ -156,41 +105,34 @@ export class DatasetVersionService {
|
||||
capability_id: "0pt.dataset.version.v1",
|
||||
run_id: `dataset-version:${input.dataset_id}:v${version.version}`,
|
||||
event_type: "COMPLETE",
|
||||
event_time: version.created_at,
|
||||
inputs: version.artifact_refs.map((ref) => ({ name: ref, ref })),
|
||||
outputs: [{ name: `dataset/${input.dataset_id}`, ref: versionRef }],
|
||||
correlation_id: input.correlation_id,
|
||||
correlation_id: version.correlation_id,
|
||||
});
|
||||
|
||||
return version;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// dataset.query
|
||||
// -----------------------------------------------------------------------
|
||||
async listVersions(datasetId: string, tenantId: string): Promise<DatasetVersion[]> {
|
||||
await this.assertDatasetTenant(datasetId, tenantId);
|
||||
return [...(this.versions.get(datasetId) ?? [])];
|
||||
return this.repo.list(datasetId, tenantId);
|
||||
}
|
||||
|
||||
async getVersion(datasetId: string, tenantId: string, version: number): Promise<DatasetVersion> {
|
||||
const list = await this.listVersions(datasetId, tenantId);
|
||||
const found = list.find((v) => v.version === version);
|
||||
if (!found) {
|
||||
throw new SiaxError("NOT_FOUND", "0pt.dataset", `dataset ${datasetId} has no version ${version}`);
|
||||
await this.assertDatasetTenant(datasetId, tenantId);
|
||||
if (!Number.isSafeInteger(version) || version < 1) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "version must be a positive integer");
|
||||
}
|
||||
return found;
|
||||
return this.repo.get(datasetId, tenantId, version);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// dataset.export
|
||||
// -----------------------------------------------------------------------
|
||||
async exportDataset(input: {
|
||||
dataset_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
version: number;
|
||||
format: DatasetExport["format"];
|
||||
/** Where the export bytes were written (by the exporting job). */
|
||||
object_location: { provider: string; bucket: string; key: string; region?: string };
|
||||
size_bytes: number;
|
||||
checksum_sha256: string;
|
||||
@@ -198,7 +140,18 @@ export class DatasetVersionService {
|
||||
correlation_id?: string;
|
||||
}): Promise<DatasetExport> {
|
||||
const version = await this.getVersion(input.dataset_id, input.tenant_id, input.version);
|
||||
const exportId = `dsexp_${crypto.randomUUID()}`;
|
||||
if (!(["parquet", "csv", "json"] as const).includes(input.format)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "unsupported export format");
|
||||
}
|
||||
if (!Number.isSafeInteger(input.size_bytes) || input.size_bytes < 0) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "size_bytes must be a non-negative safe integer");
|
||||
}
|
||||
if (!/^[a-fA-F0-9]{64}$/.test(input.checksum_sha256)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "checksum_sha256 must be a 64-character hex digest");
|
||||
}
|
||||
|
||||
const exportKey = `dsexp:${input.tenant_id}:${input.dataset_id}:v${input.version}:${input.format}:${input.checksum_sha256.toLowerCase()}`;
|
||||
const exportId = `dsexp_${crypto.createHash("sha256").update(exportKey).digest("hex").slice(0, 32)}`;
|
||||
|
||||
const artifact = await this.st0re.createArtifact({
|
||||
tenant_id: input.tenant_id,
|
||||
@@ -214,14 +167,14 @@ export class DatasetVersionService {
|
||||
? "text/csv"
|
||||
: "application/json",
|
||||
size_bytes: input.size_bytes,
|
||||
checksum_sha256: input.checksum_sha256,
|
||||
checksum_sha256: input.checksum_sha256.toLowerCase(),
|
||||
pii_level: "none",
|
||||
retention_policy: "standard",
|
||||
data_residency: "eu",
|
||||
risk_level: "L2",
|
||||
correlation_id: input.correlation_id,
|
||||
created_by: input.actor_id,
|
||||
idempotency_key: `dsexp:${input.dataset_id}:v${input.version}:${input.format}:${input.checksum_sha256}`,
|
||||
idempotency_key: exportKey,
|
||||
});
|
||||
|
||||
const artifactRef = formatResourceRef({
|
||||
@@ -229,6 +182,7 @@ export class DatasetVersionService {
|
||||
resourceType: "artifact",
|
||||
resourceId: artifact.artifact_id,
|
||||
});
|
||||
const createdAt = artifact.created_at ?? new Date().toISOString();
|
||||
const record: DatasetExport = {
|
||||
export_id: exportId,
|
||||
dataset_id: input.dataset_id,
|
||||
@@ -236,7 +190,7 @@ export class DatasetVersionService {
|
||||
tenant_id: input.tenant_id,
|
||||
artifact_ref: artifactRef,
|
||||
format: input.format,
|
||||
created_at: new Date().toISOString(),
|
||||
created_at: createdAt,
|
||||
};
|
||||
|
||||
const versionRef = formatResourceRef({
|
||||
@@ -260,8 +214,10 @@ export class DatasetVersionService {
|
||||
input_refs: [versionRef],
|
||||
output_refs: [artifactRef],
|
||||
payload: { format: input.format, size_bytes: input.size_bytes },
|
||||
created_at: record.created_at,
|
||||
created_at: createdAt,
|
||||
occurred_at: createdAt,
|
||||
correlation_id: input.correlation_id,
|
||||
idempotency_key: exportKey,
|
||||
});
|
||||
await this.lineage.emit({
|
||||
tenant_id: input.tenant_id,
|
||||
@@ -269,6 +225,7 @@ export class DatasetVersionService {
|
||||
capability_id: "0pt.dataset.export.v1",
|
||||
run_id: `dataset-export:${exportId}`,
|
||||
event_type: "COMPLETE",
|
||||
event_time: createdAt,
|
||||
inputs: [{ name: `dataset/${input.dataset_id}`, ref: versionRef }],
|
||||
outputs: [{ name: artifactRef, ref: artifactRef }],
|
||||
correlation_id: input.correlation_id,
|
||||
@@ -277,9 +234,8 @@ export class DatasetVersionService {
|
||||
return record;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
private async assertDatasetTenant(datasetId: string, tenantId: string): Promise<void> {
|
||||
if (!tenantId) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "tenant_id is required");
|
||||
if (!tenantId?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "tenant_id is required");
|
||||
let dataset;
|
||||
try {
|
||||
dataset = await this.datasets.getDataset(datasetId);
|
||||
|
||||
Reference in New Issue
Block a user