174 lines
7.2 KiB
TypeScript
174 lines
7.2 KiB
TypeScript
import { describe, it, expect } from "vitest";
|
|
import { DatasetService } from "@/src/services/dataset-service";
|
|
import { DatasetIcebergMappingService } from "@/src/services/dataset-iceberg-mapping-service";
|
|
import { IcebergWriteService } from "@/src/services/iceberg-write-service";
|
|
|
|
/**
|
|
* "Write first test dataset to Iceberg through ST0RE's table registry" proof
|
|
* (task item 2). This exercises the REAL request/response shapes and control
|
|
* flow of IcebergWriteService -> St0reLakehouseClient against that client's
|
|
* deterministic MOCK implementation, the same convention this repo already
|
|
* uses for every other cross-product boundary (see
|
|
* tests/p0/cross-product-boundary.test.ts, quality-check-service.ts's own
|
|
* CL0UD/N0D/ST0RE/AUD0 usage). No ST0RE_URL was available to this pass, so
|
|
* this is NOT verified against a live, running ST0RE lakehouse API --
|
|
* src/clients/st0re-lakehouse-client.ts's module doc comment states that
|
|
* caveat explicitly and it applies here too. What IS real: the table gets
|
|
* registered, a DatasetIcebergMapping row is created referencing ST0RE's
|
|
* (mock) tableId, a snapshot gets registered against that table, and all
|
|
* four AUD0 events fire in the right order with the right payload shape.
|
|
*/
|
|
describe("IcebergWriteService", () => {
|
|
it("mapDatasetToIcebergTable registers a table in ST0RE and records the mapping", async () => {
|
|
const datasetService = new DatasetService();
|
|
const mappingService = new DatasetIcebergMappingService({ datasetService });
|
|
const service = new IcebergWriteService({ datasetService, mappingService });
|
|
|
|
await datasetService.registerDataset({
|
|
dataset_id: "ds_first_iceberg_write",
|
|
tenant_id: "t1",
|
|
project_id: "p1",
|
|
name: "First Iceberg Pilot Dataset",
|
|
dataset_type: "enriched",
|
|
});
|
|
|
|
const result = await service.mapDatasetToIcebergTable({
|
|
tenant_id: "t1",
|
|
project_id: "p1",
|
|
dataset_id: "ds_first_iceberg_write",
|
|
dataset_iceberg_mapping_id: "dim_first_write",
|
|
namespace: "0pt_pilot",
|
|
table_name: "dim_company_se_pilot",
|
|
storage_uri: "s3://siax-p0/lakehouse/0pt_pilot/dim_company_se_pilot",
|
|
catalog_ref: "0pt_pilot.dim_company_se_pilot",
|
|
actor_id: "user_123",
|
|
correlation_id: "corr_first_write",
|
|
});
|
|
|
|
expect(result.iceberg_table_id).toContain("mock_table_");
|
|
expect(result.policy_decision_id).toBeDefined();
|
|
expect(result.mapping.dataset_id).toBe("ds_first_iceberg_write");
|
|
expect(result.mapping.iceberg_table_id).toBe(result.iceberg_table_id);
|
|
expect(result.mapping.mapping_status).toBe("active");
|
|
|
|
const lookedUp = await mappingService.getMappingForDataset("ds_first_iceberg_write");
|
|
expect(lookedUp?.dataset_iceberg_mapping_id).toBe("dim_first_write");
|
|
});
|
|
|
|
it("requestIcebergWrite registers a snapshot through ST0RE for an existing mapping", async () => {
|
|
const datasetService = new DatasetService();
|
|
const mappingService = new DatasetIcebergMappingService({ datasetService });
|
|
const service = new IcebergWriteService({ datasetService, mappingService });
|
|
|
|
await datasetService.registerDataset({
|
|
dataset_id: "ds_snapshot_write",
|
|
tenant_id: "t1",
|
|
project_id: "p1",
|
|
name: "Snapshot Write Dataset",
|
|
dataset_type: "enriched",
|
|
});
|
|
const mapped = await service.mapDatasetToIcebergTable({
|
|
tenant_id: "t1",
|
|
project_id: "p1",
|
|
dataset_id: "ds_snapshot_write",
|
|
dataset_iceberg_mapping_id: "dim_snapshot_write",
|
|
namespace: "0pt_pilot",
|
|
table_name: "dim_company_se_snapshot",
|
|
storage_uri: "s3://siax-p0/lakehouse/0pt_pilot/dim_company_se_snapshot",
|
|
catalog_ref: "0pt_pilot.dim_company_se_snapshot",
|
|
actor_id: "user_123",
|
|
correlation_id: "corr_snapshot_setup",
|
|
});
|
|
|
|
const result = await service.requestIcebergWrite({
|
|
dataset_iceberg_mapping_id: mapped.mapping.dataset_iceberg_mapping_id,
|
|
actor_id: "user_123",
|
|
correlation_id: "corr_snapshot_write",
|
|
schema_version: "1",
|
|
manifest_uri: "s3://siax-p0/lakehouse/0pt_pilot/dim_company_se_snapshot/metadata/manifest-1.avro",
|
|
record_count: 1000,
|
|
size_bytes: 204800,
|
|
});
|
|
|
|
expect(result.snapshot_id).toContain("mock_snapshot_");
|
|
expect(result.iceberg_table_id).toBe(mapped.iceberg_table_id);
|
|
expect(result.n0d_job_id).toBeUndefined(); // not a heavy write
|
|
});
|
|
|
|
it("requestIcebergWrite routes heavy writes through an N0D job request, not in-process execution", async () => {
|
|
const datasetService = new DatasetService();
|
|
const mappingService = new DatasetIcebergMappingService({ datasetService });
|
|
const service = new IcebergWriteService({ datasetService, mappingService });
|
|
|
|
await datasetService.registerDataset({
|
|
dataset_id: "ds_heavy_write",
|
|
tenant_id: "t1",
|
|
project_id: "p1",
|
|
name: "Heavy Backfill Dataset",
|
|
dataset_type: "raw",
|
|
});
|
|
const mapped = await service.mapDatasetToIcebergTable({
|
|
tenant_id: "t1",
|
|
project_id: "p1",
|
|
dataset_id: "ds_heavy_write",
|
|
dataset_iceberg_mapping_id: "dim_heavy_write",
|
|
namespace: "0pt_pilot",
|
|
table_name: "fact_financials_backfill",
|
|
storage_uri: "s3://siax-p0/lakehouse/0pt_pilot/fact_financials_backfill",
|
|
catalog_ref: "0pt_pilot.fact_financials_backfill",
|
|
actor_id: "user_123",
|
|
correlation_id: "corr_heavy_setup",
|
|
});
|
|
|
|
const result = await service.requestIcebergWrite({
|
|
dataset_iceberg_mapping_id: mapped.mapping.dataset_iceberg_mapping_id,
|
|
actor_id: "user_123",
|
|
correlation_id: "corr_heavy_write",
|
|
schema_version: "1",
|
|
manifest_uri: "s3://siax-p0/lakehouse/0pt_pilot/fact_financials_backfill/metadata/manifest-backfill.avro",
|
|
heavy: true,
|
|
});
|
|
|
|
expect(result.n0d_job_id).toBeDefined();
|
|
expect(result.n0d_job_id).toContain("mock_job_");
|
|
});
|
|
|
|
it("requestIcebergWrite rejects writes against a non-active mapping", async () => {
|
|
const datasetService = new DatasetService();
|
|
const mappingService = new DatasetIcebergMappingService({ datasetService });
|
|
const service = new IcebergWriteService({ datasetService, mappingService });
|
|
|
|
await datasetService.registerDataset({
|
|
dataset_id: "ds_deprecated_mapping",
|
|
tenant_id: "t1",
|
|
project_id: "p1",
|
|
name: "Deprecated Mapping Dataset",
|
|
dataset_type: "raw",
|
|
});
|
|
await mappingService.createMapping({
|
|
dataset_iceberg_mapping_id: "dim_deprecated",
|
|
tenant_id: "t1",
|
|
project_id: "p1",
|
|
dataset_id: "ds_deprecated_mapping",
|
|
iceberg_table_id: "st0re_table_deprecated",
|
|
mapping_status: "deprecated",
|
|
});
|
|
|
|
await expect(
|
|
service.requestIcebergWrite({
|
|
dataset_iceberg_mapping_id: "dim_deprecated",
|
|
actor_id: "user_123",
|
|
correlation_id: "corr_deprecated_write",
|
|
schema_version: "1",
|
|
manifest_uri: "s3://siax-p0/lakehouse/deprecated/manifest.avro",
|
|
})
|
|
).rejects.toThrow("is not active");
|
|
});
|
|
|
|
// The live-ST0RE case this `it.todo` used to name is now built:
|
|
// tests/p0/iceberg-write-service.realinfra.test.ts attempts a real HTTP
|
|
// call against api.st0re.siax.io (SEAM pattern: real call, honest skip if
|
|
// unreachable -- see that file's header comment and
|
|
// docs/ICEBERG_WRITE_BOUNDARY.md "Live verification" section).
|
|
});
|