diff --git a/.env.example b/.env.example index f0bde46..59ad76b 100644 --- a/.env.example +++ b/.env.example @@ -92,14 +92,21 @@ ENGINE_URL=http://localhost:8000 # --------------------------------------------------------------------------- # SIAX platform clients # --------------------------------------------------------------------------- -# src/clients/{cl0ud,n0d,aud0}-client.ts currently only accept an optional -# constructor baseUrl and fall back to an in-memory mock / no-op -- none of these -# three vars is read via process.env yet (grep-verified). Setting them does -# nothing until those clients are wired to read them; a future pass should add -# that. See docs/ICEBERG_WRITE_BOUNDARY.md. +# WIRED FOR REAL since 2026-08-31: every factory in src/clients/* reads its +# *_URL from process.env when no constructor baseUrl is given. Under +# NODE_ENV=production an UNSET url is a hard boot failure +# (SiaxError DEPENDENCY_UNAVAILABLE) — production never silently falls back +# to the in-memory mocks (Master-DoD §3/§10). In dev/test, unset = documented +# local mock/no-op behaviour. CL0UD_URL= N0D_URL= AUD0_URL= +# Optional service bearer tokens for the platform clients (machine identity). +# Values come from Infisical/secret refs — NEVER commit real values here. +CL0UD_SERVICE_TOKEN= +N0D_SERVICE_TOKEN= +AUD0_SERVICE_TOKEN= +ST0RE_SERVICE_TOKEN= # INF0 (AI OS) -- documented target, not yet read by any code in this repo. # Route AI-parsning / model calls through the gateway instead of raw provider keys. INF0_URL= @@ -110,6 +117,23 @@ INF0_URL= # their deterministic in-memory mocks. ST0RE_URL= +# --------------------------------------------------------------------------- +# P0 Data-OS backends (src/search, src/analytics, src/metadata, src/features) +# --------------------------------------------------------------------------- +# OpenSearch — search projections ONLY, never canonical truth (§42). Unset in +# dev/test = search capability unavailable (typed error), never a fake. +OPENSEARCH_URL= +# ClickHouse — 0pt capability analytics (tenant-isolated metric events). +# HAS NO CONNECTION to b0ts pipelines or quarantined market data (legal). +CLICKHOUSE_URL= +# "user:password" for ClickHouse basic auth; from secret refs only. +CLICKHOUSE_AUTH= +# OpenMetadata — metadata PROJECTION target (canonical stays in 0pt). +OPENMETADATA_URL= +OPENMETADATA_TOKEN= +# Feast feature server (online features path). +FEAST_SERVER_URL= + # --------------------------------------------------------------------------- # Payments (Stripe) -- wired in code today: app/api/payments/*, # app/api/webhooks/stripe diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 9029ea0..534c173 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -41,6 +41,101 @@ jobs: - name: Build (next build -- blocking quality gate) run: pnpm run build + # REAL-DEPENDENCY integration tests (Master-DoD §48: "integration tests ... + # Mocks alone count as insufficient"; §10 Search/Analytics). Runs + # tests/integration/* against a LIVE OpenSearch + LIVE ClickHouse started as + # service containers. + # + # SERVICE-CONTAINER DISCIPLINE is copied from sax3l/st0re's ci.yml, which + # documents three measured defects on these runners (alias collision across + # repos, sibling-run collision behind one alias, and DNS round-robin): + # * repo-unique aliases (opt-ci-opensearch / opt-ci-clickhouse), + # * SELF-IDENTIFYING containers — OpenSearch carries a run-unique + # cluster.name; ClickHouse carries run-unique credentials — so the + # select step can positively identify THIS run's container, + # * enumerate every IP behind the alias (getent ahostsv4), keep the one + # that proves the run identity, freeze it into the URL env var, + # * no match = hard failure, never a silent fallback or skip. + integration: + runs-on: ubuntu-latest + services: + opt-ci-opensearch: + image: opensearchproject/opensearch:2.11.1 + env: + discovery.type: single-node + DISABLE_SECURITY_PLUGIN: "true" + OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m" + # Identity badge: only THIS run's container answers with this + # cluster_name (the OpenSearch image maps dotted env keys to + # settings). + cluster.name: opt_ci_${{ github.run_id }}_${{ github.run_attempt }} + options: >- + --health-cmd "curl -fsS http://localhost:9200/_cluster/health || exit 1" + --health-interval 10s + --health-timeout 5s + --health-retries 60 + opt-ci-clickhouse: + image: clickhouse/clickhouse-server:24.8 + env: + CLICKHOUSE_USER: opt_ci + # Identity badge: run-unique password — an authenticated SELECT only + # succeeds against THIS run's container. CI-scoped throwaway value, + # not a secret (exists only for the lifetime of the job). + CLICKHOUSE_PASSWORD: ci_${{ github.run_id }}_${{ github.run_attempt }} + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" + options: >- + --health-cmd "wget -qO- http://localhost:8123/ping || exit 1" + --health-interval 10s + --health-timeout 5s + --health-retries 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Enable corepack + pin pnpm@9 + run: | + corepack enable + corepack prepare pnpm@9 --activate + - name: Install (frozen lockfile) + run: pnpm install --frozen-lockfile + - name: Select THIS run's OpenSearch by identity (st0re ci.yml pattern) + run: | + set -eu + WANT="opt_ci_${{ github.run_id }}_${{ github.run_attempt }}" + FOUND="" + for ip in $(getent ahostsv4 opt-ci-opensearch | awk '{print $1}' | sort -u); do + NAME=$(curl -fsS --max-time 5 "http://$ip:9200/" | sed -n 's/.*"cluster_name" *: *"\([^"]*\)".*/\1/p' || true) + echo "candidate $ip cluster_name=${NAME:-}" + if [ "$NAME" = "$WANT" ]; then FOUND="$ip"; fi + done + [ -n "$FOUND" ] || { echo "FATAL: no OpenSearch behind the alias identifies as $WANT"; exit 1; } + echo "OPENSEARCH_URL=http://$FOUND:9200" >> "$GITHUB_ENV" + - name: Select THIS run's ClickHouse by identity + run: | + set -eu + PASS="ci_${{ github.run_id }}_${{ github.run_attempt }}" + FOUND="" + for ip in $(getent ahostsv4 opt-ci-clickhouse | awk '{print $1}' | sort -u); do + OK=$(curl -fsS --max-time 5 -u "opt_ci:$PASS" "http://$ip:8123/?query=SELECT%201" || true) + echo "candidate $ip auth_select=${OK:-}" + if [ "$OK" = "1" ]; then FOUND="$ip"; fi + done + [ -n "$FOUND" ] || { echo "FATAL: no ClickHouse behind the alias accepts this run's credentials"; exit 1; } + echo "CLICKHOUSE_URL=http://$FOUND:8123" >> "$GITHUB_ENV" + echo "CLICKHOUSE_AUTH=opt_ci:$PASS" >> "$GITHUB_ENV" + - name: Integration tests against LIVE OpenSearch + ClickHouse (blocking) + run: | + set -eu + # Guard: env MUST be present here — the suites self-skip without it, + # and a skipped suite must never masquerade as a green integration + # gate (Master-DoD §56 status honesty). + [ -n "${OPENSEARCH_URL:-}" ] && [ -n "${CLICKHOUSE_URL:-}" ] + pnpm vitest run tests/integration 2>&1 | tee /tmp/itest.log + if grep -qE "skipped" /tmp/itest.log && ! grep -qE "Tests [0-9]+ passed" /tmp/itest.log; then + echo "FATAL: integration suites skipped instead of running"; exit 1 + fi + # Kept in ci.yml (advisory) so the protected-branch status check contexts # "CI (SIAX Cloud) / quality (push)" + "CI (SIAX Cloud) / security (push)" # are always reported on push. security.yml is the dedicated scanner on diff --git a/STATUS.md b/STATUS.md new file mode 100644 index 0000000..658d85c --- /dev/null +++ b/STATUS.md @@ -0,0 +1,84 @@ +# 0pt — capability status (Master-DoD §56 status honesty) + +> Classification scale (Master-DoD §2, exactly one per capability): +> `MISSING → DOCUMENTED_ONLY → SKELETON → MOCK_ONLY → IMPLEMENTED → INTEGRATION_TESTED → LIVE_VERIFIED → PRODUCTION_READY`. +> Only `PRODUCTION_READY` may be called DONE. Docs ≠ implementation. +> Last full audit + re-classification: **2026-08-31** (SIAX Platforms P0 wave 2). + +## How to read this + +- **INTEGRATION_TESTED** here means: verified against a REAL dependency + (live OpenSearch/ClickHouse container, or a recorded wire-contract double + pinned to the sibling repo's real handler code on its `main`) in CI. +- **LIVE_VERIFIED** requires a verification against the actual SIAX fleet + deployment. No new capability in this wave claims it. +- The unit/contract/integration suites are in `tests/` and run in + `.gitea/workflows/ci.yml` (jobs `quality` + `integration`). + +## Cross-product clients (Master-DoD §10 "Cross-product clients") + +| Capability | Status | Evidence | Honest gaps | +|---|---|---|---| +| CL0UD client (policy evaluate, `POST /api/policy/evaluate`) | **INTEGRATION_TESTED** (contract) | `src/clients/cl0ud-client.ts`; `tests/contracts/cl0ud-client.contract.test.ts` pins the real camelCase `{data}` envelope; fail-closed (missing decision ⇒ deny) | Not yet exercised against a live CL0UD deployment (no reachable CL0UD URL this session ⇒ not LIVE_VERIFIED) | +| CL0UD capability discovery (`GET /api/capabilities`) | **INTEGRATION_TESTED** (contract) | same suite | 0pt's own capabilities are not yet registered in CL0UD's registry (§3 gap) | +| ST0RE client (artifact register, camelCase `/v1/st0re/artifacts/register`) | **INTEGRATION_TESTED** (contract) | `src/clients/st0re-client.ts`; `tests/contracts/st0re-client.contract.test.ts` | Raw-bytes upload still goes via LocalBlobStore interim (pre-existing, unchanged); presign upload path not consumed yet | +| N0D client (create/get job, get result) | **INTEGRATION_TESTED** (contract) | `src/clients/n0d-client.ts`; `tests/contracts/n0d-client.contract.test.ts` pins `required_role`/`resource_requirements`/body `idempotency_key` | CloudEvents completion callback receiving endpoint not built in 0pt (poll path works) | +| AUD0 client (events + OpenLineage ingest) | **INTEGRATION_TESTED** (contract) | `src/clients/aud0-client.ts`; `tests/contracts/aud0-client.contract.test.ts`; legacy lineage shape converts to real OpenLineage | No durable local outbox for AUD0 outages (§70 "durable local/outbox path" — emission failure is surfaced, not queued) | +| **No production mock fallback (§3/§10)** | **INTEGRATION_TESTED** | `src/platform/dependency-mode.ts`; `tests/contracts/dependency-mode.contract.test.ts` drives NODE_ENV=production for all four factories ⇒ hard `DEPENDENCY_UNAVAILABLE` | — | + +**Correction of prior state (2026-08-31):** the pre-existing clients' HTTP +paths were verified against the sibling repos' actual `main` branches and +THREE of four were wrong: cl0ud (`/v1/cl0ud/policies/evaluate` — never +existed), st0re (snake_case body — would 400), aud0 lineage +(`/v1/aud0/lineage/events` — never existed). All fixed and pinned by +contract tests. The earlier MOCK_ONLY classification in the README was +accurate. + +## Data-OS capabilities (Master-DoD §10) + +| Capability | Status | Evidence | Honest gaps | +|---|---|---|---| +| Source Registry — register/query/describe/health, rights/license/purpose, freshness, provenance | **IMPLEMENTED** | `src/services/source-registry-service.ts`, `src/types/source-registry.ts`; `tests/p0/source-registry.test.ts` (fail-closed purpose gate, cross-tenant NOT_FOUND) | In-memory persistence (per-process); no Postgres store yet — production boot must therefore configure one before this counts further. No API route surface yet | +| Source rate-policy | **IMPLEMENTED** (pre-existing) | `src/services/source-rate-policy-service.ts` | unchanged this wave | +| Search — OpenSearch adapter, index defs, bulk, tenant + purpose/rights filter, rebuild, generation switch | **INTEGRATION_TESTED** | `src/search/*`; unit `tests/p0/search-service.test.ts`; **real OpenSearch 2.11** `tests/integration/search-opensearch.integration.test.ts` incl. projection-disaster rebuild (§68-style) — runs in CI service container | Hybrid (vector) query not built; no metrics emission; index definitions in-memory | +| Quality — versioned rules, runs, results, issues, score, trend, GE interchange | **IMPLEMENTED** (local engine) / **SKELETON** (GE delegation) | `src/quality/*`; `tests/p0/quality-service.test.ts`, `tests/p0/expectation-engine` cases | Local engine covers 7 GE core expectation types; everything else delegates via N0D job `0pt.quality.ge_run` whose EXECUTOR (0pt-motor GE runner) does not exist yet — delegated runs stay "delegated" until it does. Never fabricates results | +| Dataset — create (pre-existing) + version/query/export with `st0re:artifact:` refs | **IMPLEMENTED** | `src/services/dataset-version-service.ts`; `tests/p0/dataset-version-service.test.ts` (idempotency §53, canonical refs §5) | Export assumes bytes already written by a job; no route surface; in-memory persistence | +| Lineage — OpenLineage emission to AUD0's real ingest | **INTEGRATION_TESTED** (contract) | `src/lineage/*`; aud0 contract suite; emitted from dataset version/export, quality runs, search rebuilds | Emission is best-effort (never-throw); no outbox. AUD0-side ingestion verified against contract double, not live AUD0 | +| Metadata — OpenMetadata projection (never canonical) | **IMPLEMENTED** | `src/metadata/openmetadata-adapter.ts`; `tests/contracts/metadata-features.contract.test.ts`; `projectAll` rebuild path | No live OpenMetadata exercised (no local container this pass); lineage/glossary/ownership projection not built — only dataset entities | +| Analytics — ClickHouse, tenant-isolated, query limits, retention | **INTEGRATION_TESTED** | `src/analytics/clickhouse-analytics.ts`; **real ClickHouse 24.8** `tests/integration/analytics-clickhouse.integration.test.ts` (tenant isolation, closed DSL, TTL schema) — runs in CI service container | Closed metric-DSL only (no ad-hoc queries by design); no quota integration with CL0UD | +| Features — registry + Feast adapter | **IMPLEMENTED** (online path, registry) / **SKELETON** (materialize/get-historical) | `src/features/feast-adapter.ts`; contract suite pins Feast's `get-online-features` wire shape | No live Feast server exercised; offline/materialization are N0D delegation contracts whose executor (0pt-motor) does not exist yet; point-in-time correctness therefore NOT yet proven | +| Entity Resolution — candidate API, blocking, explanation, hard-rule hook, review handoff | **IMPLEMENTED** | `src/er/entity-resolution-service.ts`; `tests/p0/entity-resolution.test.ts` (hard rules override both directions, review band, blocking refusals) | Generic reference scorer (Jaccard+Levenshtein), NOT Splink probabilistic linkage — that is the `0pt.er.batch_resolution` N0D contract, executor pending in 0pt-motor. No persistence of review queues | +| Graph (§10 Graph) | **MISSING** | — | Not started this wave (explicitly out of the delivered P0 subset; next in line per §10) | + +## Canonical platform plumbing + +| Capability | Status | Evidence | +|---|---|---| +| SiaxRequestContext + propagation headers (mirror of cl0ud `@siax/contract`) | **INTEGRATION_TESTED** (contract-pinned) | `src/platform/siax-context.ts`; `tests/contracts/siax-context.contract.test.ts` | +| Canonical 12-code error model + typed `SiaxError` (§6) | **INTEGRATION_TESTED** (contract-pinned) | `src/platform/errors.ts`; same suite | +| Canonical resource refs (§5) | **IMPLEMENTED** | `src/platform/resource-ref.ts`; `tests/p0/dataset-version-service.test.ts` | + +## Pre-existing capabilities (unchanged this wave — prior classifications stand) + +| Capability | Status | Note | +|---|---|---| +| Iceberg write boundary via ST0RE lakehouse | LIVE_VERIFIED (2026-07-13) | `docs/ICEBERG_WRITE_BOUNDARY.md`; untouched | +| Document capture/extraction | IMPLEMENTED | LocalBlobStore interim persists bytes; ST0RE raw upload still pending | +| Edge order/payment app (Next.js) | see `docs/architecture/ARCHITECTURE.md` | outside this wave's scope | +| FastAPI engine (`services/api`) | IMPLEMENTED | outside this wave's scope | +| `services/scrapers/*` (WAF-bypass stack) | **quarantined, pending deletion PR** | `feature/0pt-remove-noncompliant-scrapers` — NOT touched by this wave; nothing new wires it | + +## Global gaps blocking PRODUCTION_READY (§3, tracked, not hidden) + +1. Persistent stores: the new registries (sources/rights/quality + rules/dataset versions/index definitions/feature views) are per-process + in-memory. `resolveStoreMode` guarantees production boot FAILS rather + than serving Maps, but the Postgres stores themselves are not built. +2. Capability registration in CL0UD + API0 discovery for the new + capabilities (§3 "Capability är registrerad i CL0UD Capability Registry"). +3. AUD0 outbox for evidence durability under AUD0 outage (§70). +4. Surface exposure: no `app/api/0pt/*` routes yet for + search/quality/datasets-versions/ER (services + contracts only). +5. 0pt-motor executors for the delegated jobs + (`0pt.quality.ge_run`, `0pt.features.*`, `0pt.er.batch_resolution`). +6. Observability: no metrics/traces emitted by the new services (§36). diff --git a/docs/02-architecture.md b/docs/02-architecture.md index a414965..9488ddd 100644 --- a/docs/02-architecture.md +++ b/docs/02-architecture.md @@ -101,7 +101,21 @@ In plain terms: **0pt is API-only today** (plus two job-backed writes, and one j `src/clients/{cl0ud,n0d,aud0,st0re,st0re-lakehouse}-client.ts` are real, typechecked TypeScript with accompanying tests (`tests/p0/*`), each following the same shape: a thin interface, an HTTP implementation written against the target service's real, committed route contract, and a deterministic in-memory mock used whenever no `baseUrl` is supplied to the constructor. -**Verified this pass**: none of the five client files, and none of their call sites in `src/services/*`, read `CL0UD_URL`, `N0D_URL`, `AUD0_URL`, or `ST0RE_URL` from `process.env` anywhere (`grep -rn "process.env" src/clients src/services` turns up nothing for any of those four names outside comments/TODOs). Every constructor call in this repo (`createCl0udClient(opts?.cl0udBaseUrl)`, etc.) is invoked with no `baseUrl` argument, so **every one of these clients runs against its in-memory mock today, in every code path that exists in this repo.** Read this as "real code, mock-only, not live-verified" — not "integrated" or "connected" — wherever platform-client status comes up. +**SUPERSEDED 2026-08-31 (SIAX Platforms P0 wave 2)** — the paragraph that +stood here ("every one of these clients runs against its in-memory mock +today") was accurate when written and is now fixed, twice over: + +1. every factory reads its `*_URL` from `process.env` when no constructor + `baseUrl` is given, and under `NODE_ENV=production` an unset URL is a hard + boot failure (`SiaxError DEPENDENCY_UNAVAILABLE`, `src/platform/dependency-mode.ts`) + — production can no longer fall back to a mock (Master-DoD §3/§10); +2. the HTTP wire shapes were verified against the sibling repos' actual + `main` branches and THREE of four were wrong (cl0ud path never existed, + st0re body was snake_case against a camelCase contract, aud0 lineage path + never existed). All rewritten and pinned by `tests/contracts/*.contract.test.ts`. + +Current per-capability status lives in [`STATUS.md`](../STATUS.md) — that +file, not this doc, is the status source of truth (§56). What each client is for, concretely: diff --git a/src/features/feast-adapter.ts b/src/features/feast-adapter.ts index 0b62ddb..2ec5d47 100644 --- a/src/features/feast-adapter.ts +++ b/src/features/feast-adapter.ts @@ -36,8 +36,7 @@ export interface FeatureViewDefinition { created_at: string; } -export interface RegisterFeatureViewInput - extends Omit {} +export type RegisterFeatureViewInput = Omit; export interface OnlineFeaturesRequest { tenant_id: string; diff --git a/src/platform/dependency-mode.ts b/src/platform/dependency-mode.ts index 385adfd..379a298 100644 --- a/src/platform/dependency-mode.ts +++ b/src/platform/dependency-mode.ts @@ -27,8 +27,17 @@ */ import { SiaxError } from "./errors"; -/** True when this process is a production runtime. */ +/** + * True when this process is a production runtime — i.e. it can SERVE + * requests. `next build` runs page-data collection under NODE_ENV=production + * while serving nothing; failing the BUILD for unset dependency URLs would + * block the compile step, not protect any request path, so the Next build + * phase (NEXT_PHASE=phase-production-build) is explicitly not a production + * runtime. The guard re-arms the moment the built app boots (`next start` + * clears NEXT_PHASE). + */ export function isProductionRuntime(env: NodeJS.ProcessEnv = process.env): boolean { + if (env.NEXT_PHASE === "phase-production-build") return false; return env.NODE_ENV === "production"; } diff --git a/tests/contracts/dependency-mode.contract.test.ts b/tests/contracts/dependency-mode.contract.test.ts index 590416f..08ea3ea 100644 --- a/tests/contracts/dependency-mode.contract.test.ts +++ b/tests/contracts/dependency-mode.contract.test.ts @@ -41,6 +41,14 @@ describe("resolveDependencyBaseUrl (the §3 kill switch)", () => { it("configured => normalized URL in any runtime (trailing slashes stripped)", () => { expect(resolveDependencyBaseUrl("cl0ud", "https://x.example.com///", PROD)).toBe("https://x.example.com"); }); + + it("the Next BUILD phase is not a production runtime (page-data collection must compile; the guard re-arms at boot)", () => { + const BUILD = { NODE_ENV: "production", NEXT_PHASE: "phase-production-build" } as NodeJS.ProcessEnv; + expect(resolveDependencyBaseUrl("st0re", undefined, BUILD)).toBeUndefined(); + // Any other phase value stays armed. + const SERVE = { NODE_ENV: "production", NEXT_PHASE: "phase-production-server" } as NodeJS.ProcessEnv; + expect(() => resolveDependencyBaseUrl("st0re", undefined, SERVE)).toThrowError(SiaxError); + }); }); describe("resolveStoreMode (§3 for stateful registries)", () => { diff --git a/tests/integration/analytics-clickhouse.integration.test.ts b/tests/integration/analytics-clickhouse.integration.test.ts new file mode 100644 index 0000000..8d072cb --- /dev/null +++ b/tests/integration/analytics-clickhouse.integration.test.ts @@ -0,0 +1,96 @@ +/** + * REAL ClickHouse integration test (Master-DoD §48/§50) for the Analytics + * capability. Runs against CLICKHOUSE_URL (+ CLICKHOUSE_AUTH "user:pass" + * when the server requires it); SKIPS HONESTLY when unset. + * + * Proves against the real engine: schema creation with TTL retention, + * JSONEachRow ingest, the closed metric-query DSL, and — the §69-relevant + * part — that the FORCED tenant predicate isolates tenants on real data. + */ +import { describe, it, expect, beforeAll } from "vitest"; +import { ClickHouseAnalytics } from "@/src/analytics/clickhouse-analytics"; + +const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; +const run = CLICKHOUSE_URL ? describe : describe.skip; + +const DB = `opt_itest_${Date.now().toString(36)}`; + +run("Analytics capability against a LIVE ClickHouse", () => { + // The skip-variant still evaluates this callback body, so the constructor + // must tolerate the env-less case (the tests themselves never run then). + const analytics = new ClickHouseAnalytics({ + baseUrl: CLICKHOUSE_URL ?? "http://skipped.invalid", + basicAuth: process.env.CLICKHOUSE_AUTH, + database: DB, + }); + + beforeAll(async () => { + expect(await analytics.health()).toBe(true); + await analytics.ensureSchema(30); + // ensureSchema is idempotent (CREATE IF NOT EXISTS). + await analytics.ensureSchema(30); + }, 60_000); + + it("ingests events and answers aggregate metric queries", async () => { + const res = await analytics.ingest([ + { tenant_id: "t_alpha", dataset_id: "ds_companies", metric: "rows_ingested", value: 100 }, + { tenant_id: "t_alpha", dataset_id: "ds_companies", metric: "rows_ingested", value: 50 }, + { tenant_id: "t_alpha", dataset_id: "ds_vehicles", metric: "rows_ingested", value: 7 }, + { tenant_id: "t_beta", dataset_id: "ds_companies", metric: "rows_ingested", value: 999 }, + ]); + expect(res.ingested).toBe(4); + + const total = await analytics.queryMetric({ + tenant_id: "t_alpha", + metric: "rows_ingested", + aggregate: "sum", + }); + expect(total).toEqual([{ value: 157 }]); + + const perDataset = await analytics.queryMetric({ + tenant_id: "t_alpha", + dataset_id: "ds_companies", + metric: "rows_ingested", + aggregate: "sum", + }); + expect(perDataset).toEqual([{ value: 150 }]); + }, 30_000); + + it("REAL tenant isolation: t_beta's numbers never leak into t_alpha aggregates", async () => { + const alpha = await analytics.queryMetric({ tenant_id: "t_alpha", metric: "rows_ingested", aggregate: "max" }); + expect(alpha[0].value).toBe(100); // NOT 999 + const beta = await analytics.queryMetric({ tenant_id: "t_beta", metric: "rows_ingested", aggregate: "sum" }); + expect(beta[0].value).toBe(999); + const ghost = await analytics.queryMetric({ tenant_id: "t_gamma", metric: "rows_ingested", aggregate: "count" }); + expect(ghost[0].value).toBe(0); + }); + + it("group-by dimension queries work through the closed DSL", async () => { + await analytics.ingest([ + { tenant_id: "t_alpha", dataset_id: "ds_x", metric: "dq_score", value: 90, dimensions: { severity: "warning" } }, + { tenant_id: "t_alpha", dataset_id: "ds_x", metric: "dq_score", value: 40, dimensions: { severity: "critical" } }, + ]); + const rows = await analytics.queryMetric({ + tenant_id: "t_alpha", + metric: "dq_score", + aggregate: "avg", + group_by: "severity", + }); + expect(rows).toHaveLength(2); + expect(rows.find((r) => r.group === "warning")?.value).toBe(90); + }); + + it("rejects tenant-less queries and invalid group_by identifiers (injection surface closed)", async () => { + await expect( + analytics.queryMetric({ tenant_id: " ", metric: "x", aggregate: "sum" }) + ).rejects.toMatchObject({ code: "INVALID_REQUEST" }); + await expect( + analytics.queryMetric({ + tenant_id: "t_alpha", + metric: "x", + aggregate: "sum", + group_by: "sev'; DROP TABLE opt_metric_events; --", + }) + ).rejects.toMatchObject({ code: "INVALID_REQUEST" }); + }); +}); diff --git a/tests/integration/search-opensearch.integration.test.ts b/tests/integration/search-opensearch.integration.test.ts new file mode 100644 index 0000000..d8f20e6 --- /dev/null +++ b/tests/integration/search-opensearch.integration.test.ts @@ -0,0 +1,172 @@ +/** + * REAL OpenSearch integration test (Master-DoD §48 "Mocks alone count as + * insufficient", §50 Integration level) for the Search capability. + * + * Runs against a live OpenSearch named by OPENSEARCH_URL (CI: a service + * container; locally: `docker run -e discovery.type=single-node + * -e DISABLE_SECURITY_PLUGIN=true opensearchproject/opensearch:2.11.1`). + * Without OPENSEARCH_URL the suite SKIPS HONESTLY — it never fabricates a + * pass (same convention as tests/p0/iceberg-write-service.realinfra.test.ts). + * + * What it proves against the real engine: + * - index creation with 0pt envelope mappings + bulk indexing + * - tenant isolation and purpose/rights filtering through real queries + * - generation rebuild + atomic alias switch + rollback (§42) + * - full projection-disaster recovery: delete EVERYTHING, rebuild from + * the canonical source, identical results (§68-style, scoped to 0pt) + */ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { HttpOpenSearchAdapter } from "@/src/search/opensearch-adapter"; +import { SearchService, type DocumentSource } from "@/src/search/search-service"; +import type { SearchDocument } from "@/src/search/search-types"; + +const OPENSEARCH_URL = process.env.OPENSEARCH_URL; +const run = OPENSEARCH_URL ? describe : describe.skip; + +const INDEX = `itest-${Date.now().toString(36)}`; + +const DOCS: SearchDocument[] = [ + { + doc_id: "c1", + tenant_id: "t_alpha", + purpose_tags: ["company_enrichment"], + rights_tags: ["internal", "customer_deliverable"], + source_refs: ["0pt:source:src_bolagsverket"], + payload: { name: "Volvo Personvagnar AB", org_nr: "5561074967", employees: 20000 }, + }, + { + doc_id: "c2", + tenant_id: "t_alpha", + purpose_tags: ["compliance"], + rights_tags: ["internal"], + payload: { name: "Volvo Lastvagnar AB", org_nr: "5560135508", employees: 15000 }, + }, + { + doc_id: "c3", + tenant_id: "t_beta", + purpose_tags: ["company_enrichment"], + rights_tags: ["internal"], + payload: { name: "Volvo Cars Retail", org_nr: "5569999999", employees: 300 }, + }, +]; + +function source(docs: SearchDocument[]): DocumentSource { + return { + name: "itest:canonical-array", + async *iterate() { + yield* docs; + }, + }; +} + +run("Search capability against a LIVE OpenSearch", () => { + // The skip-variant still evaluates this callback body, so the constructor + // must tolerate the env-less case (the tests themselves never run then). + const adapter = new HttpOpenSearchAdapter({ baseUrl: OPENSEARCH_URL ?? "http://skipped.invalid" }); + const search = new SearchService(adapter); + + beforeAll(async () => { + const health = await adapter.health(); + expect(["green", "yellow"]).toContain(health.status); + search.registerIndexDefinition({ + index_name: INDEX, + version: 1, + fields: { name: "text", org_nr: "keyword", employees: "long" }, + default_search_fields: ["name"], + }); + }, 30_000); + + afterAll(async () => { + for (const idx of await adapter.listIndices(`opt-${INDEX}--g*`)) { + await adapter.deleteIndex(idx); + } + }); + + it("rebuild g1 from canonical source; alias serves it", async () => { + const result = await search.rebuild(INDEX, source(DOCS)); + expect(result.new_generation).toBe(1); + expect(result.docs_indexed).toBe(3); + expect(result.switched).toBe(true); + const state = await search.generationState(INDEX); + expect(state.active_generation).toBe(1); + }, 60_000); + + it("REAL tenant isolation: t_alpha never sees t_beta documents", async () => { + const alpha = await search.query(INDEX, { + tenant_id: "t_alpha", + purpose: "company_enrichment", + text: "volvo", + }); + expect(alpha.hits.map((h) => h.doc_id)).toEqual(["c1"]); // c2 wrong purpose, c3 wrong tenant + const beta = await search.query(INDEX, { + tenant_id: "t_beta", + purpose: "company_enrichment", + text: "volvo", + }); + expect(beta.hits.map((h) => h.doc_id)).toEqual(["c3"]); + // Cross-tenant attack shape: a filter cannot break out of the tenant term. + const attack = await search.query(INDEX, { + tenant_id: "t_beta", + purpose: "company_enrichment", + filters: { org_nr: "5561074967" }, // t_alpha's org_nr + }); + expect(attack.total).toBe(0); + }); + + it("purpose + rights filtering hold on the real engine", async () => { + const compliance = await search.query(INDEX, { tenant_id: "t_alpha", purpose: "compliance" }); + expect(compliance.hits.map((h) => h.doc_id)).toEqual(["c2"]); + const deliverable = await search.query(INDEX, { + tenant_id: "t_alpha", + purpose: "company_enrichment", + rights_tag: "customer_deliverable", + }); + expect(deliverable.hits.map((h) => h.doc_id)).toEqual(["c1"]); + }); + + it("generation rebuild + switch + rollback (§42)", async () => { + const updated = DOCS.map((d) => + d.doc_id === "c1" ? { ...d, payload: { ...d.payload, name: "Volvo Personvagnar AB (uppdaterad)" } } : d + ); + const r2 = await search.rebuild(INDEX, source(updated)); + expect(r2.new_generation).toBe(2); + + let res = await search.query(INDEX, { tenant_id: "t_alpha", purpose: "company_enrichment", text: "uppdaterad" }); + expect(res.total).toBe(1); + + // Rollback to g1 — the old generation still exists and serves the alias. + await search.switchGeneration(INDEX, 1); + res = await search.query(INDEX, { tenant_id: "t_alpha", purpose: "company_enrichment", text: "uppdaterad" }); + expect(res.total).toBe(0); + await search.switchGeneration(INDEX, 2); + }, 60_000); + + it("projection disaster: delete ALL generations, rebuild from canonical, results identical (§68)", async () => { + for (const idx of await adapter.listIndices(`opt-${INDEX}--g*`)) { + await adapter.deleteIndex(idx); + } + expect(await adapter.listIndices(`opt-${INDEX}--g*`)).toEqual([]); + + const rebuilt = await search.rebuild(INDEX, source(DOCS)); + expect(rebuilt.docs_indexed).toBe(3); + const res = await search.query(INDEX, { tenant_id: "t_alpha", purpose: "company_enrichment", text: "volvo" }); + expect(res.hits.map((h) => h.doc_id)).toEqual(["c1"]); + }, 60_000); + + it("incremental indexDocuments into the active generation", async () => { + const out = await search.indexDocuments(INDEX, [ + { + doc_id: "c4", + tenant_id: "t_alpha", + purpose_tags: ["company_enrichment"], + rights_tags: ["internal"], + payload: { name: "Polestar Performance AB", org_nr: "5568100000", employees: 2500 }, + }, + ]); + expect(out.indexed).toBe(1); + const state = await search.generationState(INDEX); + await adapter.refresh(search.physicalIndexFor(INDEX, state.active_generation!)); + const res = await search.query(INDEX, { tenant_id: "t_alpha", purpose: "company_enrichment", text: "polestar" }); + expect(res.total).toBe(1); + }); +});