Re-runs this session's task spec against 0pt's actual repo state, which
turned out to already have a substantial, tested implementation from a
prior pass (35b6ec6): dataset<->Iceberg-table mapping, source rate policy,
and an ST0RE lakehouse client that was honestly marked "not live-verified".
This pass found api.st0re.siax.io IS reachable from this environment and
used it -- first via curl to safely discover the real contract, then
through 0pt's own updated code -- to drive a real namespace -> table ->
snapshot write against production ST0RE (task item 3's "write ONE first
test dataset" proof). That surfaced and fixed three real bugs the
mock-only pass could not have caught:
- ST0RE requires X-Tenant-Id/X-Actor-Id headers on every lakehouse route;
the previous HTTP client sent none. Fixed via a threaded
St0reRequestContext.
- IcebergTableInput.datasetId is a foreign key into ST0RE's OWN dataset
table, not a free-form string -- sending 0pt's dataset_id 400s live with
a Postgres FK violation. Fixed by no longer sending it; 0pt's
dataset<->table linkage lives entirely in DatasetIcebergMapping.
- ST0RE enforces a real unique constraint on lakehouse namespaces
(tenant_id, project_id, name) with no upsert route. Fixed with a
client-side lookup-or-create (listNamespaces + findOrRegisterNamespace).
Also closed a field-level gap against the task's literal schema spec:
0pt.dataset_iceberg_mapping.schema.v1 now carries namespace_id (a real
reference to ST0RE's LakehouseNamespace.namespaceId, not guessed) and
mapped_at, both additive alongside the existing namespace/created_at
fields (sql/005, JSON schema, types, service).
tests/p0/iceberg-write-service.realinfra.test.ts replaces the prior
it.todo with a real SEAM-pattern live test: attempts the real HTTP call,
skips honestly if ST0RE is unreachable, never fabricates success. It ran
successfully against production ST0RE this session (real namespace/table/
snapshot UUIDs logged in docs/ICEBERG_WRITE_BOUNDARY.md).
Two ST0RE-side cross-tenant read bugs were found and flagged (not fixed
here -- out of scope for this repo): GET .../namespaces ignores its own
tenant header and query filters; GET .../tables/:id doesn't check the
caller's tenant against the table's owner.
npm test: 155 passed, 0 failed, 0 todo (up from 154 passed + 1 todo).
npx tsc --noEmit: 1 pre-existing, unrelated error in
quality-report-service.ts (present before this pass, part of the wider
uncommitted P0 data-platform build already in this repo) -- not touched.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
121 lines
5.5 KiB
TypeScript
121 lines
5.5 KiB
TypeScript
import { describe, it, expect, beforeAll } from "vitest";
|
|
import { DatasetService } from "@/src/services/dataset-service";
|
|
import { DatasetIcebergMappingService } from "@/src/services/dataset-iceberg-mapping-service";
|
|
import { IcebergWriteService } from "@/src/services/iceberg-write-service";
|
|
|
|
/**
|
|
* LIVE ST0RE integration -- the SEAM pattern this session's task spec asked
|
|
* for: "attempt the real call, degrade honestly with a clear status if ST0RE
|
|
* is not reachable, never fabricate success" (same convention this same
|
|
* session verified in api0's clients, and the same convention ST0RE's own
|
|
* `integration-realinfra.test.ts` uses for its DATABASE_URL/MINIO_* gate).
|
|
*
|
|
* This is NOT a mock-client test -- it constructs `IcebergWriteService` with
|
|
* a real `st0reBaseUrl` and drives the actual `HttpSt0reLakehouseClient`
|
|
* HTTP path against production ST0RE (api.st0re.siax.io, currently reachable
|
|
* from this environment -- DNS for that name resolves here even though
|
|
* Cloudflare's public A record is still pending per this session's known
|
|
* infra state; set ST0RE_URL to override).
|
|
*
|
|
* Real writes this test produces land in ST0RE's live Postgres under a
|
|
* dedicated pilot tenant (`t_0pt_realinfra_pilot` by default, overridable via
|
|
* ST0RE_TEST_TENANT_ID) -- that IS the point of task item 3 ("write ONE
|
|
* first test dataset to an Iceberg table through ST0RE's table registry").
|
|
*
|
|
* Honest degrade: a `beforeAll` reachability probe (5s timeout) decides
|
|
* whether to run for real or `ctx.skip()` with a logged reason. This test
|
|
* never asserts success without having actually made the HTTP calls, and
|
|
* never falls back to the mock client silently.
|
|
*/
|
|
const ST0RE_URL = process.env.ST0RE_URL ?? "https://api.st0re.siax.io";
|
|
const TEST_TENANT = process.env.ST0RE_TEST_TENANT_ID ?? "t_0pt_realinfra_pilot";
|
|
const TEST_PROJECT = "p_0pt_realinfra";
|
|
const TEST_ACTOR = "0pt-realinfra-test";
|
|
|
|
let reachable = false;
|
|
let unreachableReason = "";
|
|
|
|
beforeAll(async () => {
|
|
try {
|
|
const res = await fetch(`${ST0RE_URL}/v1/st0re/lakehouse/namespaces`, {
|
|
headers: { "X-Tenant-Id": TEST_TENANT, "X-Actor-Id": TEST_ACTOR },
|
|
signal: AbortSignal.timeout(5000),
|
|
});
|
|
reachable = res.ok;
|
|
if (!reachable) unreachableReason = `ST0RE responded HTTP ${res.status}`;
|
|
} catch (err) {
|
|
reachable = false;
|
|
unreachableReason = err instanceof Error ? err.message : String(err);
|
|
}
|
|
}, 10000);
|
|
|
|
describe("IcebergWriteService against a LIVE ST0RE deployment", () => {
|
|
it(
|
|
"mapDatasetToIcebergTable + requestIcebergWrite register a real namespace/table/snapshot in ST0RE",
|
|
async (ctx) => {
|
|
if (!reachable) {
|
|
console.warn(
|
|
`[realinfra] ST0RE (${ST0RE_URL}) not reachable this run (${unreachableReason}) -- skipping honestly, not fabricating success.`
|
|
);
|
|
ctx.skip();
|
|
return;
|
|
}
|
|
|
|
const datasetService = new DatasetService();
|
|
const mappingService = new DatasetIcebergMappingService({ datasetService });
|
|
const service = new IcebergWriteService({ datasetService, mappingService, st0reBaseUrl: ST0RE_URL });
|
|
|
|
const stamp = Date.now();
|
|
const datasetId = `ds_realinfra_pilot_${stamp}`;
|
|
await datasetService.registerDataset({
|
|
dataset_id: datasetId,
|
|
tenant_id: TEST_TENANT,
|
|
project_id: TEST_PROJECT,
|
|
name: "0pt real-ST0RE pilot dataset",
|
|
dataset_type: "enriched",
|
|
});
|
|
|
|
const mapped = await service.mapDatasetToIcebergTable({
|
|
tenant_id: TEST_TENANT,
|
|
project_id: TEST_PROJECT,
|
|
dataset_id: datasetId,
|
|
dataset_iceberg_mapping_id: `dim_realinfra_${stamp}`,
|
|
namespace: "0pt_pilot",
|
|
table_name: `dim_company_se_pilot_${stamp}`,
|
|
storage_uri: `s3://siax-tenants/prod/tenants/${TEST_TENANT}/projects/${TEST_PROJECT}/lakehouse/0pt_pilot/dim_company_se_pilot_${stamp}`,
|
|
catalog_ref: `0pt_pilot.dim_company_se_pilot_${stamp}`,
|
|
actor_id: TEST_ACTOR,
|
|
correlation_id: `corr_realinfra_${stamp}`,
|
|
});
|
|
|
|
// Real ST0RE issues Postgres UUIDs -- never the mock client's
|
|
// "mock_table_<timestamp>" shape. This is the assertion that proves
|
|
// this test actually hit the live HTTP path, not the mock fallback.
|
|
expect(mapped.iceberg_table_id).not.toContain("mock_table_");
|
|
expect(mapped.iceberg_table_id).toMatch(/^[0-9a-f-]{36}$/i);
|
|
expect(mapped.namespace_id).not.toContain("mock_ns_");
|
|
expect(mapped.namespace_id).toMatch(/^[0-9a-f-]{36}$/i);
|
|
expect(mapped.mapping.dataset_id).toBe(datasetId);
|
|
// eslint-disable-next-line no-console
|
|
console.log(`[realinfra] LIVE ST0RE table registered: tableId=${mapped.iceberg_table_id} namespaceId=${mapped.namespace_id}`);
|
|
|
|
const write = await service.requestIcebergWrite({
|
|
dataset_iceberg_mapping_id: mapped.mapping.dataset_iceberg_mapping_id,
|
|
actor_id: TEST_ACTOR,
|
|
correlation_id: `corr_realinfra_write_${stamp}`,
|
|
schema_version: "1",
|
|
manifest_uri: `s3://siax-tenants/prod/tenants/${TEST_TENANT}/projects/${TEST_PROJECT}/lakehouse/0pt_pilot/dim_company_se_pilot_${stamp}/metadata/manifest-1.avro`,
|
|
record_count: 1,
|
|
size_bytes: 128,
|
|
});
|
|
|
|
expect(write.snapshot_id).not.toContain("mock_snapshot_");
|
|
expect(write.snapshot_id).toMatch(/^[0-9a-f-]{36}$/i);
|
|
expect(write.iceberg_table_id).toBe(mapped.iceberg_table_id);
|
|
// eslint-disable-next-line no-console
|
|
console.log(`[realinfra] LIVE ST0RE snapshot registered: snapshotId=${write.snapshot_id}`);
|
|
},
|
|
20000
|
|
);
|
|
});
|