diff --git a/src/services/dataset-version-service.ts b/src/services/dataset-version-service.ts index 2c1afbf..9852710 100644 --- a/src/services/dataset-version-service.ts +++ b/src/services/dataset-version-service.ts @@ -1,96 +1,62 @@ /** - * src/services/dataset-version-service.ts - * ======================================== - * P0 Dataset capability extensions (Master-DoD §10 Dataset: - * dataset.create/version/query/export) layered ON TOP of DatasetService (the - * owner of 0pt.dataset.schema.v1 registry entries — deliberately untouched: - * riv inte fungerande kod). + * Dataset version/query/export capability. * - * dataset.version -> createVersion: an immutable, monotonically numbered - * version whose artifacts are CANONICAL ST0RE REFS - * (`st0re:artifact:art_x` — Master-DoD §5); 0pt never - * stores bytes here, only references. - * dataset.query -> listVersions/getVersion (tenant-gated). - * dataset.export -> exportDataset: registers the export artifact in ST0RE - * (real client), records AUD0 evidence + OpenLineage, - * returns the canonical `st0re:artifact:...` ref. - * - * Idempotency (§53): createVersion accepts an idempotency_key; the same key - * returns the SAME version instead of minting a duplicate. + * Dataset metadata is canonical in DatasetService; immutable version metadata + * is persisted through DatasetVersionRepository; bytes remain canonical ST0RE + * artifacts. AUD0/OpenLineage record the evidence chain. */ import crypto from "crypto"; -import { SiaxError } from "../platform/errors"; -import { formatResourceRef, isResourceRef } from "../platform/resource-ref"; import { createAud0Client, type Aud0Client } from "../clients/aud0-client"; import { createSt0reClient, type St0reClient } from "../clients/st0re-client"; import { LineageEmitter } from "../lineage/lineage-emitter"; +import { + createDatasetVersionRepository, + type DatasetVersionRepository, +} from "../repositories/dataset-version-repository"; +import { SiaxError } from "../platform/errors"; +import { formatResourceRef, isResourceRef } from "../platform/resource-ref"; +import type { + CreateDatasetVersionInput, + DatasetExport, + DatasetVersion, +} from "../types/dataset-version"; import { DatasetService } from "./dataset-service"; -export interface DatasetVersion { - dataset_id: string; - version: number; - tenant_id: string; - project_id: string; - /** Canonical ST0RE artifact refs (st0re:artifact:...) making up this version. */ - artifact_refs: string[]; - /** Schema version of the dataset content at this version. */ - schema_version: string; - /** Row/record count when known. */ - record_count?: number; - description?: string; - created_at: string; - created_by: string; - correlation_id?: string; -} +export type { CreateDatasetVersionInput, DatasetExport, DatasetVersion } from "../types/dataset-version"; -export interface CreateDatasetVersionInput { - dataset_id: string; - tenant_id: string; - project_id: string; - artifact_refs: string[]; - schema_version: string; - record_count?: number; - description?: string; - created_by: string; - correlation_id?: string; - idempotency_key?: string; -} - -export interface DatasetExport { - export_id: string; - dataset_id: string; - version: number; - tenant_id: string; - /** Canonical ref of the registered export artifact. */ - artifact_ref: string; - format: "parquet" | "csv" | "json"; - created_at: string; +export interface DatasetVersionServiceOptions { + aud0BaseUrl?: string; + st0reBaseUrl?: string; + databaseUrl?: string; + datasetService?: DatasetService; + repository?: DatasetVersionRepository; } export class DatasetVersionService { - private readonly versions = new Map(); - private readonly idempotency = new Map(); + private readonly repo: DatasetVersionRepository; private readonly aud0: Aud0Client; private readonly st0re: St0reClient; private readonly lineage: LineageEmitter; readonly datasets: DatasetService; - constructor(opts?: { aud0BaseUrl?: string; st0reBaseUrl?: string; datasetService?: DatasetService }) { + constructor(opts?: DatasetVersionServiceOptions) { this.aud0 = createAud0Client(opts?.aud0BaseUrl); this.st0re = createSt0reClient(opts?.st0reBaseUrl); this.lineage = new LineageEmitter(this.aud0); - this.datasets = opts?.datasetService ?? new DatasetService(opts?.aud0BaseUrl); + this.datasets = opts?.datasetService ?? new DatasetService({ + aud0BaseUrl: opts?.aud0BaseUrl, + databaseUrl: opts?.databaseUrl, + }); + this.repo = opts?.repository ?? createDatasetVersionRepository(opts?.databaseUrl); } - // ----------------------------------------------------------------------- - // dataset.version - // ----------------------------------------------------------------------- async createVersion(input: CreateDatasetVersionInput): Promise { await this.assertDatasetTenant(input.dataset_id, input.tenant_id); - - if (input.idempotency_key) { - const existing = this.idempotency.get(`${input.tenant_id}:${input.idempotency_key}`); - if (existing) return existing; + if (!input.project_id?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "project_id is required"); + if (!input.schema_version?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "schema_version is required"); + if (!input.created_by?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "created_by is required"); + if (input.record_count !== undefined && (!Number.isSafeInteger(input.record_count) || input.record_count < 0)) { + throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "record_count must be a non-negative safe integer"); } if (input.artifact_refs.length === 0) { throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "a dataset version needs at least one artifact ref"); @@ -100,37 +66,19 @@ export class DatasetVersionService { throw new SiaxError( "INVALID_REQUEST", "0pt.dataset", - `artifact ref "${ref}" is not a canonical st0re:artifact:... reference (Master-DoD §5)` + `artifact ref "${ref}" is not a canonical st0re:artifact:... reference (Master-DoD §5)`, ); } } - const list = this.versions.get(input.dataset_id) ?? []; - const version: DatasetVersion = { - dataset_id: input.dataset_id, - version: (list[list.length - 1]?.version ?? 0) + 1, - tenant_id: input.tenant_id, - project_id: input.project_id, - artifact_refs: [...input.artifact_refs], - schema_version: input.schema_version, - record_count: input.record_count, - description: input.description, - created_at: new Date().toISOString(), - created_by: input.created_by, - correlation_id: input.correlation_id, - }; - list.push(version); - this.versions.set(input.dataset_id, list); - if (input.idempotency_key) { - this.idempotency.set(`${input.tenant_id}:${input.idempotency_key}`, version); - } - + const version = await this.repo.create(input); const versionRef = formatResourceRef({ product: "0pt", resourceType: "dataset", resourceId: input.dataset_id, version: String(version.version), }); + await this.aud0.ingestEvent({ event_id: `evt_0pt_dataset_version_${input.dataset_id}_v${version.version}`, event_type: "0pt.dataset_version_created", @@ -145,10 +93,11 @@ export class DatasetVersionService { result: "success", input_refs: version.artifact_refs, output_refs: [versionRef], - payload: { schema_version: input.schema_version, record_count: input.record_count }, + payload: { schema_version: version.schema_version, record_count: version.record_count }, created_at: version.created_at, - correlation_id: input.correlation_id, - idempotency_key: input.idempotency_key, + occurred_at: version.created_at, + correlation_id: version.correlation_id, + idempotency_key: `dataset-version:${input.tenant_id}:${input.dataset_id}:v${version.version}`, }); await this.lineage.emit({ tenant_id: input.tenant_id, @@ -156,41 +105,34 @@ export class DatasetVersionService { capability_id: "0pt.dataset.version.v1", run_id: `dataset-version:${input.dataset_id}:v${version.version}`, event_type: "COMPLETE", + event_time: version.created_at, inputs: version.artifact_refs.map((ref) => ({ name: ref, ref })), outputs: [{ name: `dataset/${input.dataset_id}`, ref: versionRef }], - correlation_id: input.correlation_id, + correlation_id: version.correlation_id, }); return version; } - // ----------------------------------------------------------------------- - // dataset.query - // ----------------------------------------------------------------------- async listVersions(datasetId: string, tenantId: string): Promise { await this.assertDatasetTenant(datasetId, tenantId); - return [...(this.versions.get(datasetId) ?? [])]; + return this.repo.list(datasetId, tenantId); } async getVersion(datasetId: string, tenantId: string, version: number): Promise { - const list = await this.listVersions(datasetId, tenantId); - const found = list.find((v) => v.version === version); - if (!found) { - throw new SiaxError("NOT_FOUND", "0pt.dataset", `dataset ${datasetId} has no version ${version}`); + await this.assertDatasetTenant(datasetId, tenantId); + if (!Number.isSafeInteger(version) || version < 1) { + throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "version must be a positive integer"); } - return found; + return this.repo.get(datasetId, tenantId, version); } - // ----------------------------------------------------------------------- - // dataset.export - // ----------------------------------------------------------------------- async exportDataset(input: { dataset_id: string; tenant_id: string; project_id: string; version: number; format: DatasetExport["format"]; - /** Where the export bytes were written (by the exporting job). */ object_location: { provider: string; bucket: string; key: string; region?: string }; size_bytes: number; checksum_sha256: string; @@ -198,7 +140,18 @@ export class DatasetVersionService { correlation_id?: string; }): Promise { const version = await this.getVersion(input.dataset_id, input.tenant_id, input.version); - const exportId = `dsexp_${crypto.randomUUID()}`; + if (!(["parquet", "csv", "json"] as const).includes(input.format)) { + throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "unsupported export format"); + } + if (!Number.isSafeInteger(input.size_bytes) || input.size_bytes < 0) { + throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "size_bytes must be a non-negative safe integer"); + } + if (!/^[a-fA-F0-9]{64}$/.test(input.checksum_sha256)) { + throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "checksum_sha256 must be a 64-character hex digest"); + } + + const exportKey = `dsexp:${input.tenant_id}:${input.dataset_id}:v${input.version}:${input.format}:${input.checksum_sha256.toLowerCase()}`; + const exportId = `dsexp_${crypto.createHash("sha256").update(exportKey).digest("hex").slice(0, 32)}`; const artifact = await this.st0re.createArtifact({ tenant_id: input.tenant_id, @@ -214,14 +167,14 @@ export class DatasetVersionService { ? "text/csv" : "application/json", size_bytes: input.size_bytes, - checksum_sha256: input.checksum_sha256, + checksum_sha256: input.checksum_sha256.toLowerCase(), pii_level: "none", retention_policy: "standard", data_residency: "eu", risk_level: "L2", correlation_id: input.correlation_id, created_by: input.actor_id, - idempotency_key: `dsexp:${input.dataset_id}:v${input.version}:${input.format}:${input.checksum_sha256}`, + idempotency_key: exportKey, }); const artifactRef = formatResourceRef({ @@ -229,6 +182,7 @@ export class DatasetVersionService { resourceType: "artifact", resourceId: artifact.artifact_id, }); + const createdAt = artifact.created_at ?? new Date().toISOString(); const record: DatasetExport = { export_id: exportId, dataset_id: input.dataset_id, @@ -236,7 +190,7 @@ export class DatasetVersionService { tenant_id: input.tenant_id, artifact_ref: artifactRef, format: input.format, - created_at: new Date().toISOString(), + created_at: createdAt, }; const versionRef = formatResourceRef({ @@ -260,8 +214,10 @@ export class DatasetVersionService { input_refs: [versionRef], output_refs: [artifactRef], payload: { format: input.format, size_bytes: input.size_bytes }, - created_at: record.created_at, + created_at: createdAt, + occurred_at: createdAt, correlation_id: input.correlation_id, + idempotency_key: exportKey, }); await this.lineage.emit({ tenant_id: input.tenant_id, @@ -269,6 +225,7 @@ export class DatasetVersionService { capability_id: "0pt.dataset.export.v1", run_id: `dataset-export:${exportId}`, event_type: "COMPLETE", + event_time: createdAt, inputs: [{ name: `dataset/${input.dataset_id}`, ref: versionRef }], outputs: [{ name: artifactRef, ref: artifactRef }], correlation_id: input.correlation_id, @@ -277,9 +234,8 @@ export class DatasetVersionService { return record; } - // ----------------------------------------------------------------------- private async assertDatasetTenant(datasetId: string, tenantId: string): Promise { - if (!tenantId) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "tenant_id is required"); + if (!tenantId?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "tenant_id is required"); let dataset; try { dataset = await this.datasets.getDataset(datasetId);