Files
0pt/src/services/dataset-version-service.ts
T

250 lines
9.6 KiB
TypeScript

/**
* Dataset version/query/export capability.
*
* Dataset metadata is canonical in DatasetService; immutable version metadata
* is persisted through DatasetVersionRepository; bytes remain canonical ST0RE
* artifacts. AUD0/OpenLineage record the evidence chain.
*/
import crypto from "crypto";
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import { createSt0reClient, type St0reClient } from "../clients/st0re-client";
import { LineageEmitter } from "../lineage/lineage-emitter";
import {
createDatasetVersionRepository,
type DatasetVersionRepository,
} from "../repositories/dataset-version-repository";
import { SiaxError } from "../platform/errors";
import { formatResourceRef, isResourceRef } from "../platform/resource-ref";
import type {
CreateDatasetVersionInput,
DatasetExport,
DatasetVersion,
} from "../types/dataset-version";
import { DatasetService } from "./dataset-service";
export type { CreateDatasetVersionInput, DatasetExport, DatasetVersion } from "../types/dataset-version";
export interface DatasetVersionServiceOptions {
aud0BaseUrl?: string;
st0reBaseUrl?: string;
databaseUrl?: string;
datasetService?: DatasetService;
repository?: DatasetVersionRepository;
}
export class DatasetVersionService {
private readonly repo: DatasetVersionRepository;
private readonly aud0: Aud0Client;
private readonly st0re: St0reClient;
private readonly lineage: LineageEmitter;
readonly datasets: DatasetService;
constructor(opts?: DatasetVersionServiceOptions) {
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.st0re = createSt0reClient(opts?.st0reBaseUrl);
this.lineage = new LineageEmitter(this.aud0);
this.datasets = opts?.datasetService ?? new DatasetService({
aud0BaseUrl: opts?.aud0BaseUrl,
databaseUrl: opts?.databaseUrl,
});
this.repo = opts?.repository ?? createDatasetVersionRepository(opts?.databaseUrl);
}
async createVersion(input: CreateDatasetVersionInput): Promise<DatasetVersion> {
await this.assertDatasetTenant(input.dataset_id, input.tenant_id);
if (!input.project_id?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "project_id is required");
if (!input.schema_version?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "schema_version is required");
if (!input.created_by?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "created_by is required");
if (input.record_count !== undefined && (!Number.isSafeInteger(input.record_count) || input.record_count < 0)) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "record_count must be a non-negative safe integer");
}
if (input.artifact_refs.length === 0) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "a dataset version needs at least one artifact ref");
}
for (const ref of input.artifact_refs) {
if (!isResourceRef(ref, "st0re", "artifact")) {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.dataset",
`artifact ref "${ref}" is not a canonical st0re:artifact:... reference (Master-DoD §5)`,
);
}
}
const version = await this.repo.create(input);
const versionRef = formatResourceRef({
product: "0pt",
resourceType: "dataset",
resourceId: input.dataset_id,
version: String(version.version),
});
await this.aud0.ingestEvent({
event_id: `evt_0pt_dataset_version_${input.dataset_id}_v${version.version}`,
event_type: "0pt.dataset_version_created",
tenant_id: input.tenant_id,
project_id: input.project_id,
actor_id: input.created_by,
app_id: "0pt",
capability_id: "0pt.dataset.version.v1",
resource_type: "dataset_version",
resource_id: `${input.dataset_id}@${version.version}`,
risk_level: "L2",
result: "success",
input_refs: version.artifact_refs,
output_refs: [versionRef],
payload: { schema_version: version.schema_version, record_count: version.record_count },
created_at: version.created_at,
occurred_at: version.created_at,
correlation_id: version.correlation_id,
idempotency_key: `dataset-version:${input.tenant_id}:${input.dataset_id}:v${version.version}`,
});
await this.lineage.emit({
tenant_id: input.tenant_id,
project_id: input.project_id,
capability_id: "0pt.dataset.version.v1",
run_id: `dataset-version:${input.dataset_id}:v${version.version}`,
event_type: "COMPLETE",
event_time: version.created_at,
inputs: version.artifact_refs.map((ref) => ({ name: ref, ref })),
outputs: [{ name: `dataset/${input.dataset_id}`, ref: versionRef }],
correlation_id: version.correlation_id,
});
return version;
}
async listVersions(datasetId: string, tenantId: string): Promise<DatasetVersion[]> {
await this.assertDatasetTenant(datasetId, tenantId);
return this.repo.list(datasetId, tenantId);
}
async getVersion(datasetId: string, tenantId: string, version: number): Promise<DatasetVersion> {
await this.assertDatasetTenant(datasetId, tenantId);
if (!Number.isSafeInteger(version) || version < 1) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "version must be a positive integer");
}
return this.repo.get(datasetId, tenantId, version);
}
async exportDataset(input: {
dataset_id: string;
tenant_id: string;
project_id: string;
version: number;
format: DatasetExport["format"];
object_location: { provider: string; bucket: string; key: string; region?: string };
size_bytes: number;
checksum_sha256: string;
actor_id: string;
correlation_id?: string;
}): Promise<DatasetExport> {
const version = await this.getVersion(input.dataset_id, input.tenant_id, input.version);
if (!(["parquet", "csv", "json"] as const).includes(input.format)) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "unsupported export format");
}
if (!Number.isSafeInteger(input.size_bytes) || input.size_bytes < 0) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "size_bytes must be a non-negative safe integer");
}
if (!/^[a-fA-F0-9]{64}$/.test(input.checksum_sha256)) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "checksum_sha256 must be a 64-character hex digest");
}
const exportKey = `dsexp:${input.tenant_id}:${input.dataset_id}:v${input.version}:${input.format}:${input.checksum_sha256.toLowerCase()}`;
const exportId = `dsexp_${crypto.createHash("sha256").update(exportKey).digest("hex").slice(0, 32)}`;
const artifact = await this.st0re.createArtifact({
tenant_id: input.tenant_id,
project_id: input.project_id,
artifact_type: "dataset_export",
app_id: "0pt",
produced_by: "0pt.dataset.export.v1",
object_location: input.object_location,
content_type:
input.format === "parquet"
? "application/vnd.apache.parquet"
: input.format === "csv"
? "text/csv"
: "application/json",
size_bytes: input.size_bytes,
checksum_sha256: input.checksum_sha256.toLowerCase(),
pii_level: "none",
retention_policy: "standard",
data_residency: "eu",
risk_level: "L2",
correlation_id: input.correlation_id,
created_by: input.actor_id,
idempotency_key: exportKey,
});
const artifactRef = formatResourceRef({
product: "st0re",
resourceType: "artifact",
resourceId: artifact.artifact_id,
});
const createdAt = artifact.created_at ?? new Date().toISOString();
const record: DatasetExport = {
export_id: exportId,
dataset_id: input.dataset_id,
version: input.version,
tenant_id: input.tenant_id,
artifact_ref: artifactRef,
format: input.format,
created_at: createdAt,
};
const versionRef = formatResourceRef({
product: "0pt",
resourceType: "dataset",
resourceId: input.dataset_id,
version: String(version.version),
});
await this.aud0.ingestEvent({
event_id: `evt_0pt_dataset_export_${exportId}`,
event_type: "0pt.dataset_exported",
tenant_id: input.tenant_id,
project_id: input.project_id,
actor_id: input.actor_id,
app_id: "0pt",
capability_id: "0pt.dataset.export.v1",
resource_type: "dataset_export",
resource_id: exportId,
risk_level: "L3",
result: "success",
input_refs: [versionRef],
output_refs: [artifactRef],
payload: { format: input.format, size_bytes: input.size_bytes },
created_at: createdAt,
occurred_at: createdAt,
correlation_id: input.correlation_id,
idempotency_key: exportKey,
});
await this.lineage.emit({
tenant_id: input.tenant_id,
project_id: input.project_id,
capability_id: "0pt.dataset.export.v1",
run_id: `dataset-export:${exportId}`,
event_type: "COMPLETE",
event_time: createdAt,
inputs: [{ name: `dataset/${input.dataset_id}`, ref: versionRef }],
outputs: [{ name: artifactRef, ref: artifactRef }],
correlation_id: input.correlation_id,
});
return record;
}
private async assertDatasetTenant(datasetId: string, tenantId: string): Promise<void> {
if (!tenantId?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "tenant_id is required");
let dataset;
try {
dataset = await this.datasets.getDataset(datasetId);
} catch {
throw new SiaxError("NOT_FOUND", "0pt.dataset", `dataset ${datasetId} not found`);
}
if (dataset.tenant_id !== tenantId) {
throw new SiaxError("NOT_FOUND", "0pt.dataset", `dataset ${datasetId} not found`);
}
}
}