- Source Registry (§10): rights/license/purpose metadata, provenance,
freshness SLA, health checks, describe/query; purpose gate FAILS CLOSED
(no rights record => deny); cross-tenant probes => NOT_FOUND.
- Search (§10/§42): OpenSearch REST adapter (no SDK) + SearchService with
MANDATORY tenant + purpose/rights filtering, versioned index definitions,
bulk, generation-per-rebuild (opt-{name}--g{n}) with atomic alias cutover,
rollback, failed-rebuild generation discard. OpenSearch never canonical.
- Quality (§10): versioned rule registry (GE-JSON interchange), local
engine for the GE core subset (never silently passes unsupported types),
runs/results/issues/score/trend, whole-run delegation to the GE backend
via N0D (0pt.quality.ge_run) - never fabricated results.
- Dataset (§10/§5/§53): immutable numbered versions holding canonical
st0re:artifact refs only, idempotent createVersion, export registers the
ST0RE artifact + AUD0 evidence + OpenLineage.
- Metadata (§10): OpenMetadata PUT /api/v1/tables projection adapter +
projectAll rebuild path; projection never canonical.
- Analytics (§10): ClickHouse HTTP adapter, closed metric-query DSL with
FORCED tenant predicate + readonly/max_execution_time/max_result_rows,
TTL retention. Zero connection to b0ts/quarantined pipelines.
- Features (§10): versioned feature-view registry, real Feast
get-online-features adapter, materialize/get-historical as N0D delegation.
- ER (§10): generic candidate API with deterministic blocking + transparent
scoring + per-field explanations, product hard-rule hook (force both
directions), review-band handoff, Splink batch via N0D contract.
93 new unit/contract tests; suite 248 passed / 1 skipped.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
176 lines
5.9 KiB
TypeScript
176 lines
5.9 KiB
TypeScript
/**
|
|
* DatasetVersionService unit tests (Master-DoD §10 Dataset:
|
|
* create/version/query/export with canonical ST0RE artifact refs, §5 + §53).
|
|
*/
|
|
import { describe, it, expect } from "vitest";
|
|
import { DatasetVersionService } from "@/src/services/dataset-version-service";
|
|
import { parseResourceRef, formatResourceRef, isResourceRef } from "@/src/platform/resource-ref";
|
|
|
|
async function withDataset() {
|
|
const svc = new DatasetVersionService();
|
|
await svc.datasets.registerDataset({
|
|
dataset_id: "ds_companies",
|
|
tenant_id: "t_a",
|
|
project_id: "p_1",
|
|
name: "Companies",
|
|
dataset_type: "curated",
|
|
});
|
|
return svc;
|
|
}
|
|
|
|
describe("resource refs (Master-DoD §5)", () => {
|
|
it("formats and parses canonical refs incl. versions", () => {
|
|
expect(formatResourceRef({ product: "st0re", resourceType: "artifact", resourceId: "art_x" })).toBe(
|
|
"st0re:artifact:art_x"
|
|
);
|
|
expect(parseResourceRef("0pt:dataset:ds_x@3")).toEqual({
|
|
product: "0pt",
|
|
resourceType: "dataset",
|
|
resourceId: "ds_x",
|
|
version: "3",
|
|
});
|
|
expect(isResourceRef("st0re:artifact:art_x", "st0re", "artifact")).toBe(true);
|
|
expect(isResourceRef("st0re:artifact:art_x", "aud0")).toBe(false);
|
|
expect(parseResourceRef("not a ref")).toBeUndefined();
|
|
expect(parseResourceRef("db-primary-key-42")).toBeUndefined();
|
|
});
|
|
});
|
|
|
|
describe("DatasetVersionService — dataset.version", () => {
|
|
it("creates monotonically numbered versions holding canonical st0re refs", async () => {
|
|
const svc = await withDataset();
|
|
const v1 = await svc.createVersion({
|
|
dataset_id: "ds_companies",
|
|
tenant_id: "t_a",
|
|
project_id: "p_1",
|
|
artifact_refs: ["st0re:artifact:art_1"],
|
|
schema_version: "v1",
|
|
record_count: 100,
|
|
created_by: "actor_1",
|
|
});
|
|
const v2 = await svc.createVersion({
|
|
dataset_id: "ds_companies",
|
|
tenant_id: "t_a",
|
|
project_id: "p_1",
|
|
artifact_refs: ["st0re:artifact:art_2", "st0re:artifact:art_3"],
|
|
schema_version: "v1",
|
|
created_by: "actor_1",
|
|
});
|
|
expect(v1.version).toBe(1);
|
|
expect(v2.version).toBe(2);
|
|
expect((await svc.listVersions("ds_companies", "t_a")).map((v) => v.version)).toEqual([1, 2]);
|
|
expect((await svc.getVersion("ds_companies", "t_a", 2)).artifact_refs).toHaveLength(2);
|
|
});
|
|
|
|
it("REFUSES non-canonical artifact refs (no raw ids, no foreign products)", async () => {
|
|
const svc = await withDataset();
|
|
for (const bad of ["art_raw_id", "aud0:evidence:ev_1", "st0re:dataset:x", ""]) {
|
|
await expect(
|
|
svc.createVersion({
|
|
dataset_id: "ds_companies",
|
|
tenant_id: "t_a",
|
|
project_id: "p_1",
|
|
artifact_refs: [bad],
|
|
schema_version: "v1",
|
|
created_by: "actor_1",
|
|
})
|
|
).rejects.toMatchObject({ code: "INVALID_REQUEST" });
|
|
}
|
|
await expect(
|
|
svc.createVersion({
|
|
dataset_id: "ds_companies",
|
|
tenant_id: "t_a",
|
|
project_id: "p_1",
|
|
artifact_refs: [],
|
|
schema_version: "v1",
|
|
created_by: "actor_1",
|
|
})
|
|
).rejects.toMatchObject({ code: "INVALID_REQUEST" });
|
|
});
|
|
|
|
it("idempotency (§53): the same idempotency_key returns the SAME version", async () => {
|
|
const svc = await withDataset();
|
|
const a = await svc.createVersion({
|
|
dataset_id: "ds_companies",
|
|
tenant_id: "t_a",
|
|
project_id: "p_1",
|
|
artifact_refs: ["st0re:artifact:art_1"],
|
|
schema_version: "v1",
|
|
created_by: "actor_1",
|
|
idempotency_key: "retry-1",
|
|
});
|
|
const b = await svc.createVersion({
|
|
dataset_id: "ds_companies",
|
|
tenant_id: "t_a",
|
|
project_id: "p_1",
|
|
artifact_refs: ["st0re:artifact:art_1"],
|
|
schema_version: "v1",
|
|
created_by: "actor_1",
|
|
idempotency_key: "retry-1",
|
|
});
|
|
expect(b.version).toBe(a.version);
|
|
expect(await svc.listVersions("ds_companies", "t_a")).toHaveLength(1);
|
|
});
|
|
|
|
it("tenant gate: foreign tenant sees NOT_FOUND on version reads/writes (§50)", async () => {
|
|
const svc = await withDataset();
|
|
await expect(svc.listVersions("ds_companies", "t_b")).rejects.toMatchObject({ code: "NOT_FOUND" });
|
|
await expect(
|
|
svc.createVersion({
|
|
dataset_id: "ds_companies",
|
|
tenant_id: "t_b",
|
|
project_id: "p_1",
|
|
artifact_refs: ["st0re:artifact:art_1"],
|
|
schema_version: "v1",
|
|
created_by: "intruder",
|
|
})
|
|
).rejects.toMatchObject({ code: "NOT_FOUND" });
|
|
});
|
|
});
|
|
|
|
describe("DatasetVersionService — dataset.export", () => {
|
|
it("registers the export artifact via ST0RE and returns its canonical ref", async () => {
|
|
const svc = await withDataset();
|
|
await svc.createVersion({
|
|
dataset_id: "ds_companies",
|
|
tenant_id: "t_a",
|
|
project_id: "p_1",
|
|
artifact_refs: ["st0re:artifact:art_1"],
|
|
schema_version: "v1",
|
|
created_by: "actor_1",
|
|
});
|
|
const exp = await svc.exportDataset({
|
|
dataset_id: "ds_companies",
|
|
tenant_id: "t_a",
|
|
project_id: "p_1",
|
|
version: 1,
|
|
format: "parquet",
|
|
object_location: { provider: "minio", bucket: "siax-tenants", key: "t_a/exports/companies.parquet" },
|
|
size_bytes: 4096,
|
|
checksum_sha256: "b".repeat(64),
|
|
actor_id: "actor_1",
|
|
});
|
|
// Test runtime uses the ST0RE mock => mock_artifact_ prefix; the point is
|
|
// the CANONICAL REF shape, which is runtime-independent.
|
|
expect(isResourceRef(exp.artifact_ref, "st0re", "artifact")).toBe(true);
|
|
expect(exp.format).toBe("parquet");
|
|
});
|
|
|
|
it("export of a nonexistent version is NOT_FOUND", async () => {
|
|
const svc = await withDataset();
|
|
await expect(
|
|
svc.exportDataset({
|
|
dataset_id: "ds_companies",
|
|
tenant_id: "t_a",
|
|
project_id: "p_1",
|
|
version: 9,
|
|
format: "csv",
|
|
object_location: { provider: "minio", bucket: "b", key: "k" },
|
|
size_bytes: 1,
|
|
checksum_sha256: "c".repeat(64),
|
|
actor_id: "actor_1",
|
|
})
|
|
).rejects.toMatchObject({ code: "NOT_FOUND" });
|
|
});
|
|
});
|