250 lines
9.6 KiB
TypeScript
250 lines
9.6 KiB
TypeScript
/**
|
|
* Dataset version/query/export capability.
|
|
*
|
|
* Dataset metadata is canonical in DatasetService; immutable version metadata
|
|
* is persisted through DatasetVersionRepository; bytes remain canonical ST0RE
|
|
* artifacts. AUD0/OpenLineage record the evidence chain.
|
|
*/
|
|
import crypto from "crypto";
|
|
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
|
import { createSt0reClient, type St0reClient } from "../clients/st0re-client";
|
|
import { LineageEmitter } from "../lineage/lineage-emitter";
|
|
import {
|
|
createDatasetVersionRepository,
|
|
type DatasetVersionRepository,
|
|
} from "../repositories/dataset-version-repository";
|
|
import { SiaxError } from "../platform/errors";
|
|
import { formatResourceRef, isResourceRef } from "../platform/resource-ref";
|
|
import type {
|
|
CreateDatasetVersionInput,
|
|
DatasetExport,
|
|
DatasetVersion,
|
|
} from "../types/dataset-version";
|
|
import { DatasetService } from "./dataset-service";
|
|
|
|
export type { CreateDatasetVersionInput, DatasetExport, DatasetVersion } from "../types/dataset-version";
|
|
|
|
export interface DatasetVersionServiceOptions {
|
|
aud0BaseUrl?: string;
|
|
st0reBaseUrl?: string;
|
|
databaseUrl?: string;
|
|
datasetService?: DatasetService;
|
|
repository?: DatasetVersionRepository;
|
|
}
|
|
|
|
export class DatasetVersionService {
|
|
private readonly repo: DatasetVersionRepository;
|
|
private readonly aud0: Aud0Client;
|
|
private readonly st0re: St0reClient;
|
|
private readonly lineage: LineageEmitter;
|
|
readonly datasets: DatasetService;
|
|
|
|
constructor(opts?: DatasetVersionServiceOptions) {
|
|
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
|
this.st0re = createSt0reClient(opts?.st0reBaseUrl);
|
|
this.lineage = new LineageEmitter(this.aud0);
|
|
this.datasets = opts?.datasetService ?? new DatasetService({
|
|
aud0BaseUrl: opts?.aud0BaseUrl,
|
|
databaseUrl: opts?.databaseUrl,
|
|
});
|
|
this.repo = opts?.repository ?? createDatasetVersionRepository(opts?.databaseUrl);
|
|
}
|
|
|
|
async createVersion(input: CreateDatasetVersionInput): Promise<DatasetVersion> {
|
|
await this.assertDatasetTenant(input.dataset_id, input.tenant_id);
|
|
if (!input.project_id?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "project_id is required");
|
|
if (!input.schema_version?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "schema_version is required");
|
|
if (!input.created_by?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "created_by is required");
|
|
if (input.record_count !== undefined && (!Number.isSafeInteger(input.record_count) || input.record_count < 0)) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "record_count must be a non-negative safe integer");
|
|
}
|
|
if (input.artifact_refs.length === 0) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "a dataset version needs at least one artifact ref");
|
|
}
|
|
for (const ref of input.artifact_refs) {
|
|
if (!isResourceRef(ref, "st0re", "artifact")) {
|
|
throw new SiaxError(
|
|
"INVALID_REQUEST",
|
|
"0pt.dataset",
|
|
`artifact ref "${ref}" is not a canonical st0re:artifact:... reference (Master-DoD §5)`,
|
|
);
|
|
}
|
|
}
|
|
|
|
const version = await this.repo.create(input);
|
|
const versionRef = formatResourceRef({
|
|
product: "0pt",
|
|
resourceType: "dataset",
|
|
resourceId: input.dataset_id,
|
|
version: String(version.version),
|
|
});
|
|
|
|
await this.aud0.ingestEvent({
|
|
event_id: `evt_0pt_dataset_version_${input.dataset_id}_v${version.version}`,
|
|
event_type: "0pt.dataset_version_created",
|
|
tenant_id: input.tenant_id,
|
|
project_id: input.project_id,
|
|
actor_id: input.created_by,
|
|
app_id: "0pt",
|
|
capability_id: "0pt.dataset.version.v1",
|
|
resource_type: "dataset_version",
|
|
resource_id: `${input.dataset_id}@${version.version}`,
|
|
risk_level: "L2",
|
|
result: "success",
|
|
input_refs: version.artifact_refs,
|
|
output_refs: [versionRef],
|
|
payload: { schema_version: version.schema_version, record_count: version.record_count },
|
|
created_at: version.created_at,
|
|
occurred_at: version.created_at,
|
|
correlation_id: version.correlation_id,
|
|
idempotency_key: `dataset-version:${input.tenant_id}:${input.dataset_id}:v${version.version}`,
|
|
});
|
|
await this.lineage.emit({
|
|
tenant_id: input.tenant_id,
|
|
project_id: input.project_id,
|
|
capability_id: "0pt.dataset.version.v1",
|
|
run_id: `dataset-version:${input.dataset_id}:v${version.version}`,
|
|
event_type: "COMPLETE",
|
|
event_time: version.created_at,
|
|
inputs: version.artifact_refs.map((ref) => ({ name: ref, ref })),
|
|
outputs: [{ name: `dataset/${input.dataset_id}`, ref: versionRef }],
|
|
correlation_id: version.correlation_id,
|
|
});
|
|
|
|
return version;
|
|
}
|
|
|
|
async listVersions(datasetId: string, tenantId: string): Promise<DatasetVersion[]> {
|
|
await this.assertDatasetTenant(datasetId, tenantId);
|
|
return this.repo.list(datasetId, tenantId);
|
|
}
|
|
|
|
async getVersion(datasetId: string, tenantId: string, version: number): Promise<DatasetVersion> {
|
|
await this.assertDatasetTenant(datasetId, tenantId);
|
|
if (!Number.isSafeInteger(version) || version < 1) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "version must be a positive integer");
|
|
}
|
|
return this.repo.get(datasetId, tenantId, version);
|
|
}
|
|
|
|
async exportDataset(input: {
|
|
dataset_id: string;
|
|
tenant_id: string;
|
|
project_id: string;
|
|
version: number;
|
|
format: DatasetExport["format"];
|
|
object_location: { provider: string; bucket: string; key: string; region?: string };
|
|
size_bytes: number;
|
|
checksum_sha256: string;
|
|
actor_id: string;
|
|
correlation_id?: string;
|
|
}): Promise<DatasetExport> {
|
|
const version = await this.getVersion(input.dataset_id, input.tenant_id, input.version);
|
|
if (!(["parquet", "csv", "json"] as const).includes(input.format)) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "unsupported export format");
|
|
}
|
|
if (!Number.isSafeInteger(input.size_bytes) || input.size_bytes < 0) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "size_bytes must be a non-negative safe integer");
|
|
}
|
|
if (!/^[a-fA-F0-9]{64}$/.test(input.checksum_sha256)) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "checksum_sha256 must be a 64-character hex digest");
|
|
}
|
|
|
|
const exportKey = `dsexp:${input.tenant_id}:${input.dataset_id}:v${input.version}:${input.format}:${input.checksum_sha256.toLowerCase()}`;
|
|
const exportId = `dsexp_${crypto.createHash("sha256").update(exportKey).digest("hex").slice(0, 32)}`;
|
|
|
|
const artifact = await this.st0re.createArtifact({
|
|
tenant_id: input.tenant_id,
|
|
project_id: input.project_id,
|
|
artifact_type: "dataset_export",
|
|
app_id: "0pt",
|
|
produced_by: "0pt.dataset.export.v1",
|
|
object_location: input.object_location,
|
|
content_type:
|
|
input.format === "parquet"
|
|
? "application/vnd.apache.parquet"
|
|
: input.format === "csv"
|
|
? "text/csv"
|
|
: "application/json",
|
|
size_bytes: input.size_bytes,
|
|
checksum_sha256: input.checksum_sha256.toLowerCase(),
|
|
pii_level: "none",
|
|
retention_policy: "standard",
|
|
data_residency: "eu",
|
|
risk_level: "L2",
|
|
correlation_id: input.correlation_id,
|
|
created_by: input.actor_id,
|
|
idempotency_key: exportKey,
|
|
});
|
|
|
|
const artifactRef = formatResourceRef({
|
|
product: "st0re",
|
|
resourceType: "artifact",
|
|
resourceId: artifact.artifact_id,
|
|
});
|
|
const createdAt = artifact.created_at ?? new Date().toISOString();
|
|
const record: DatasetExport = {
|
|
export_id: exportId,
|
|
dataset_id: input.dataset_id,
|
|
version: input.version,
|
|
tenant_id: input.tenant_id,
|
|
artifact_ref: artifactRef,
|
|
format: input.format,
|
|
created_at: createdAt,
|
|
};
|
|
|
|
const versionRef = formatResourceRef({
|
|
product: "0pt",
|
|
resourceType: "dataset",
|
|
resourceId: input.dataset_id,
|
|
version: String(version.version),
|
|
});
|
|
await this.aud0.ingestEvent({
|
|
event_id: `evt_0pt_dataset_export_${exportId}`,
|
|
event_type: "0pt.dataset_exported",
|
|
tenant_id: input.tenant_id,
|
|
project_id: input.project_id,
|
|
actor_id: input.actor_id,
|
|
app_id: "0pt",
|
|
capability_id: "0pt.dataset.export.v1",
|
|
resource_type: "dataset_export",
|
|
resource_id: exportId,
|
|
risk_level: "L3",
|
|
result: "success",
|
|
input_refs: [versionRef],
|
|
output_refs: [artifactRef],
|
|
payload: { format: input.format, size_bytes: input.size_bytes },
|
|
created_at: createdAt,
|
|
occurred_at: createdAt,
|
|
correlation_id: input.correlation_id,
|
|
idempotency_key: exportKey,
|
|
});
|
|
await this.lineage.emit({
|
|
tenant_id: input.tenant_id,
|
|
project_id: input.project_id,
|
|
capability_id: "0pt.dataset.export.v1",
|
|
run_id: `dataset-export:${exportId}`,
|
|
event_type: "COMPLETE",
|
|
event_time: createdAt,
|
|
inputs: [{ name: `dataset/${input.dataset_id}`, ref: versionRef }],
|
|
outputs: [{ name: artifactRef, ref: artifactRef }],
|
|
correlation_id: input.correlation_id,
|
|
});
|
|
|
|
return record;
|
|
}
|
|
|
|
private async assertDatasetTenant(datasetId: string, tenantId: string): Promise<void> {
|
|
if (!tenantId?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "tenant_id is required");
|
|
let dataset;
|
|
try {
|
|
dataset = await this.datasets.getDataset(datasetId);
|
|
} catch {
|
|
throw new SiaxError("NOT_FOUND", "0pt.dataset", `dataset ${datasetId} not found`);
|
|
}
|
|
if (dataset.tenant_id !== tenantId) {
|
|
throw new SiaxError("NOT_FOUND", "0pt.dataset", `dataset ${datasetId} not found`);
|
|
}
|
|
}
|
|
}
|