Re-runs this session's task spec against 0pt's actual repo state, which
turned out to already have a substantial, tested implementation from a
prior pass (35b6ec6): dataset<->Iceberg-table mapping, source rate policy,
and an ST0RE lakehouse client that was honestly marked "not live-verified".
This pass found api.st0re.siax.io IS reachable from this environment and
used it -- first via curl to safely discover the real contract, then
through 0pt's own updated code -- to drive a real namespace -> table ->
snapshot write against production ST0RE (task item 3's "write ONE first
test dataset" proof). That surfaced and fixed three real bugs the
mock-only pass could not have caught:
- ST0RE requires X-Tenant-Id/X-Actor-Id headers on every lakehouse route;
the previous HTTP client sent none. Fixed via a threaded
St0reRequestContext.
- IcebergTableInput.datasetId is a foreign key into ST0RE's OWN dataset
table, not a free-form string -- sending 0pt's dataset_id 400s live with
a Postgres FK violation. Fixed by no longer sending it; 0pt's
dataset<->table linkage lives entirely in DatasetIcebergMapping.
- ST0RE enforces a real unique constraint on lakehouse namespaces
(tenant_id, project_id, name) with no upsert route. Fixed with a
client-side lookup-or-create (listNamespaces + findOrRegisterNamespace).
Also closed a field-level gap against the task's literal schema spec:
0pt.dataset_iceberg_mapping.schema.v1 now carries namespace_id (a real
reference to ST0RE's LakehouseNamespace.namespaceId, not guessed) and
mapped_at, both additive alongside the existing namespace/created_at
fields (sql/005, JSON schema, types, service).
tests/p0/iceberg-write-service.realinfra.test.ts replaces the prior
it.todo with a real SEAM-pattern live test: attempts the real HTTP call,
skips honestly if ST0RE is unreachable, never fabricates success. It ran
successfully against production ST0RE this session (real namespace/table/
snapshot UUIDs logged in docs/ICEBERG_WRITE_BOUNDARY.md).
Two ST0RE-side cross-tenant read bugs were found and flagged (not fixed
here -- out of scope for this repo): GET .../namespaces ignores its own
tenant header and query filters; GET .../tables/:id doesn't check the
caller's tenant against the table's owner.
npm test: 155 passed, 0 failed, 0 todo (up from 154 passed + 1 todo).
npx tsc --noEmit: 1 pre-existing, unrelated error in
quality-report-service.ts (present before this pass, part of the wider
uncommitted P0 data-platform build already in this repo) -- not touched.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
174 lines
7.2 KiB
TypeScript
174 lines
7.2 KiB
TypeScript
import { describe, it, expect } from "vitest";
|
|
import { DatasetService } from "@/src/services/dataset-service";
|
|
import { DatasetIcebergMappingService } from "@/src/services/dataset-iceberg-mapping-service";
|
|
import { IcebergWriteService } from "@/src/services/iceberg-write-service";
|
|
|
|
/**
|
|
* "Write first test dataset to Iceberg through ST0RE's table registry" proof
|
|
* (task item 2). This exercises the REAL request/response shapes and control
|
|
* flow of IcebergWriteService -> St0reLakehouseClient against that client's
|
|
* deterministic MOCK implementation, the same convention this repo already
|
|
* uses for every other cross-product boundary (see
|
|
* tests/p0/cross-product-boundary.test.ts, quality-check-service.ts's own
|
|
* CL0UD/N0D/ST0RE/AUD0 usage). No ST0RE_URL was available to this pass, so
|
|
* this is NOT verified against a live, running ST0RE lakehouse API --
|
|
* src/clients/st0re-lakehouse-client.ts's module doc comment states that
|
|
* caveat explicitly and it applies here too. What IS real: the table gets
|
|
* registered, a DatasetIcebergMapping row is created referencing ST0RE's
|
|
* (mock) tableId, a snapshot gets registered against that table, and all
|
|
* four AUD0 events fire in the right order with the right payload shape.
|
|
*/
|
|
describe("IcebergWriteService", () => {
|
|
it("mapDatasetToIcebergTable registers a table in ST0RE and records the mapping", async () => {
|
|
const datasetService = new DatasetService();
|
|
const mappingService = new DatasetIcebergMappingService({ datasetService });
|
|
const service = new IcebergWriteService({ datasetService, mappingService });
|
|
|
|
await datasetService.registerDataset({
|
|
dataset_id: "ds_first_iceberg_write",
|
|
tenant_id: "t1",
|
|
project_id: "p1",
|
|
name: "First Iceberg Pilot Dataset",
|
|
dataset_type: "enriched",
|
|
});
|
|
|
|
const result = await service.mapDatasetToIcebergTable({
|
|
tenant_id: "t1",
|
|
project_id: "p1",
|
|
dataset_id: "ds_first_iceberg_write",
|
|
dataset_iceberg_mapping_id: "dim_first_write",
|
|
namespace: "0pt_pilot",
|
|
table_name: "dim_company_se_pilot",
|
|
storage_uri: "s3://siax-p0/lakehouse/0pt_pilot/dim_company_se_pilot",
|
|
catalog_ref: "0pt_pilot.dim_company_se_pilot",
|
|
actor_id: "user_123",
|
|
correlation_id: "corr_first_write",
|
|
});
|
|
|
|
expect(result.iceberg_table_id).toContain("mock_table_");
|
|
expect(result.policy_decision_id).toBeDefined();
|
|
expect(result.mapping.dataset_id).toBe("ds_first_iceberg_write");
|
|
expect(result.mapping.iceberg_table_id).toBe(result.iceberg_table_id);
|
|
expect(result.mapping.mapping_status).toBe("active");
|
|
|
|
const lookedUp = await mappingService.getMappingForDataset("ds_first_iceberg_write");
|
|
expect(lookedUp?.dataset_iceberg_mapping_id).toBe("dim_first_write");
|
|
});
|
|
|
|
it("requestIcebergWrite registers a snapshot through ST0RE for an existing mapping", async () => {
|
|
const datasetService = new DatasetService();
|
|
const mappingService = new DatasetIcebergMappingService({ datasetService });
|
|
const service = new IcebergWriteService({ datasetService, mappingService });
|
|
|
|
await datasetService.registerDataset({
|
|
dataset_id: "ds_snapshot_write",
|
|
tenant_id: "t1",
|
|
project_id: "p1",
|
|
name: "Snapshot Write Dataset",
|
|
dataset_type: "enriched",
|
|
});
|
|
const mapped = await service.mapDatasetToIcebergTable({
|
|
tenant_id: "t1",
|
|
project_id: "p1",
|
|
dataset_id: "ds_snapshot_write",
|
|
dataset_iceberg_mapping_id: "dim_snapshot_write",
|
|
namespace: "0pt_pilot",
|
|
table_name: "dim_company_se_snapshot",
|
|
storage_uri: "s3://siax-p0/lakehouse/0pt_pilot/dim_company_se_snapshot",
|
|
catalog_ref: "0pt_pilot.dim_company_se_snapshot",
|
|
actor_id: "user_123",
|
|
correlation_id: "corr_snapshot_setup",
|
|
});
|
|
|
|
const result = await service.requestIcebergWrite({
|
|
dataset_iceberg_mapping_id: mapped.mapping.dataset_iceberg_mapping_id,
|
|
actor_id: "user_123",
|
|
correlation_id: "corr_snapshot_write",
|
|
schema_version: "1",
|
|
manifest_uri: "s3://siax-p0/lakehouse/0pt_pilot/dim_company_se_snapshot/metadata/manifest-1.avro",
|
|
record_count: 1000,
|
|
size_bytes: 204800,
|
|
});
|
|
|
|
expect(result.snapshot_id).toContain("mock_snapshot_");
|
|
expect(result.iceberg_table_id).toBe(mapped.iceberg_table_id);
|
|
expect(result.n0d_job_id).toBeUndefined(); // not a heavy write
|
|
});
|
|
|
|
it("requestIcebergWrite routes heavy writes through an N0D job request, not in-process execution", async () => {
|
|
const datasetService = new DatasetService();
|
|
const mappingService = new DatasetIcebergMappingService({ datasetService });
|
|
const service = new IcebergWriteService({ datasetService, mappingService });
|
|
|
|
await datasetService.registerDataset({
|
|
dataset_id: "ds_heavy_write",
|
|
tenant_id: "t1",
|
|
project_id: "p1",
|
|
name: "Heavy Backfill Dataset",
|
|
dataset_type: "raw",
|
|
});
|
|
const mapped = await service.mapDatasetToIcebergTable({
|
|
tenant_id: "t1",
|
|
project_id: "p1",
|
|
dataset_id: "ds_heavy_write",
|
|
dataset_iceberg_mapping_id: "dim_heavy_write",
|
|
namespace: "0pt_pilot",
|
|
table_name: "fact_financials_backfill",
|
|
storage_uri: "s3://siax-p0/lakehouse/0pt_pilot/fact_financials_backfill",
|
|
catalog_ref: "0pt_pilot.fact_financials_backfill",
|
|
actor_id: "user_123",
|
|
correlation_id: "corr_heavy_setup",
|
|
});
|
|
|
|
const result = await service.requestIcebergWrite({
|
|
dataset_iceberg_mapping_id: mapped.mapping.dataset_iceberg_mapping_id,
|
|
actor_id: "user_123",
|
|
correlation_id: "corr_heavy_write",
|
|
schema_version: "1",
|
|
manifest_uri: "s3://siax-p0/lakehouse/0pt_pilot/fact_financials_backfill/metadata/manifest-backfill.avro",
|
|
heavy: true,
|
|
});
|
|
|
|
expect(result.n0d_job_id).toBeDefined();
|
|
expect(result.n0d_job_id).toContain("mock_job_");
|
|
});
|
|
|
|
it("requestIcebergWrite rejects writes against a non-active mapping", async () => {
|
|
const datasetService = new DatasetService();
|
|
const mappingService = new DatasetIcebergMappingService({ datasetService });
|
|
const service = new IcebergWriteService({ datasetService, mappingService });
|
|
|
|
await datasetService.registerDataset({
|
|
dataset_id: "ds_deprecated_mapping",
|
|
tenant_id: "t1",
|
|
project_id: "p1",
|
|
name: "Deprecated Mapping Dataset",
|
|
dataset_type: "raw",
|
|
});
|
|
await mappingService.createMapping({
|
|
dataset_iceberg_mapping_id: "dim_deprecated",
|
|
tenant_id: "t1",
|
|
project_id: "p1",
|
|
dataset_id: "ds_deprecated_mapping",
|
|
iceberg_table_id: "st0re_table_deprecated",
|
|
mapping_status: "deprecated",
|
|
});
|
|
|
|
await expect(
|
|
service.requestIcebergWrite({
|
|
dataset_iceberg_mapping_id: "dim_deprecated",
|
|
actor_id: "user_123",
|
|
correlation_id: "corr_deprecated_write",
|
|
schema_version: "1",
|
|
manifest_uri: "s3://siax-p0/lakehouse/deprecated/manifest.avro",
|
|
})
|
|
).rejects.toThrow("is not active");
|
|
});
|
|
|
|
// The live-ST0RE case this `it.todo` used to name is now built:
|
|
// tests/p0/iceberg-write-service.realinfra.test.ts attempts a real HTTP
|
|
// call against api.st0re.siax.io (SEAM pattern: real call, honest skip if
|
|
// unreachable -- see that file's header comment and
|
|
// docs/ICEBERG_WRITE_BOUNDARY.md "Live verification" section).
|
|
});
|