Merge main into the Golden Standard scaffold branch — keep only CODEOWNERS
CI (SIAX Cloud) / integration (push) Successful in 29s
CI (SIAX Cloud) / security (push) Successful in 23s
CI (SIAX Cloud) / quality (push) Successful in 53s
CI (SIAX Cloud) / integration (pull_request) Successful in 32s
CI (SIAX Cloud) / security (pull_request) Successful in 24s
CI (SIAX Cloud) / quality (pull_request) Successful in 53s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 21s
Security Scan (SIAX Cloud) / security (push) Successful in 21s
CI (SIAX Cloud) / integration (push) Successful in 29s
CI (SIAX Cloud) / security (push) Successful in 23s
CI (SIAX Cloud) / quality (push) Successful in 53s
CI (SIAX Cloud) / integration (pull_request) Successful in 32s
CI (SIAX Cloud) / security (pull_request) Successful in 24s
CI (SIAX Cloud) / quality (pull_request) Successful in 53s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 21s
Security Scan (SIAX Cloud) / security (push) Successful in 21s
PR #21 was batch-closed 2026-09-04 as "verified against main (added-files
present: all)". That check cannot have run: `.gitea/CODEOWNERS` and
`docs/compliance/SCAFFOLD-TODO.md` were absent from main, and the other two
files differed. Reopened; this merge resolves it against what main says today.
Resolution, file by file, measured against origin/main (08c88f39):
- .devcontainer/devcontainer.json — KEEP MAIN. Main's version is strictly
richer (corepack pin, siax-doctor postStart, forwardPorts, AUTH_ENFORCE).
The scaffold's three-line version would be a regression.
- compose.dev.yaml — KEEP MAIN. Main declares a real pgvector service with
an init mount, healthcheck and a documented P014 verification. The
scaffold's `services: {}` would delete a working local dev stack.
- docs/compliance/SCAFFOLD-TODO.md — DROPPED. Written 2026-08-11 and now
false on every substantive claim: it says siax.repo.json fails schema
validation (main's has repo/lifecycle/gitHost/consumedBy/children/
operationalState and object-form domains), that domains are strings (they
are objects), that consumedBy is missing (present), and that DOC-07 has no
proven restore (docs/runbook/backup-restore.md line 23 records an executed
drill dated 2026-09-03). Merging it would add stale false statements.
- .gitea/CODEOWNERS — KEPT. Genuinely absent from main, still wanted by
CI-14, and correct as written.
Net diff against main is one added file.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,12 @@
|
||||
# BACKLOG P6 (CI-16): lets a developer run `act` to execute .gitea/workflows
|
||||
# (GitHub-Actions-compatible syntax) locally before pushing, instead of
|
||||
# discovering a workflow bug only after a Gitea runner picks it up.
|
||||
# https://github.com/nektos/act
|
||||
#
|
||||
# ubuntu-latest here maps to a real Docker image (act's default is a minimal
|
||||
# image missing tools this repo's workflows expect, e.g. curl for the
|
||||
# Trivy/gitleaks install steps) -- catthehacker's full image matches what
|
||||
# GitHub-hosted (and this fleet's siax-runner-*) ubuntu-latest actually has.
|
||||
-P ubuntu-latest=catthehacker/ubuntu:full-latest
|
||||
-W .gitea/workflows
|
||||
--container-architecture linux/amd64
|
||||
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* BACKLOG P6 (ARK-14): import-boundary check. Advisory today (run manually /
|
||||
* `pnpm run deps:check`) -- not wired as a blocking CI gate in this pass,
|
||||
* so a first real run surfacing findings doesn't turn a green PR red
|
||||
* unexpectedly. Wiring it into CI is a reasonable next step once the
|
||||
* initial finding set (if any) has been triaged.
|
||||
*
|
||||
* Boundaries encoded, matching CLAUDE.md / docs/architecture/ARCHITECTURE.md:
|
||||
* - `services/scrapers` and `services/api` are Python -- not traversed by
|
||||
* this JS/TS dependency graph at all; no rule needed, they're simply
|
||||
* out of dependency-cruiser's file-type scope.
|
||||
* - `src/` (the capability layer, blocking-typechecked) must not import
|
||||
* from `app/` or `components/` (the v0-generated UI layer, advisory-only)
|
||||
* -- that would pull UI-layer assumptions into the tested backend layer.
|
||||
* - Nothing may import from `tests/` (test helpers stay test-only).
|
||||
*/
|
||||
module.exports = {
|
||||
forbidden: [
|
||||
{
|
||||
name: 'no-src-importing-app-or-components',
|
||||
comment:
|
||||
'src/ is the blocking-typechecked capability layer (tsconfig.check.json); app/ and components/ are the advisory v0-generated UI layer. src/ importing from them would pull unverified UI-layer code into the tested backend layer.',
|
||||
severity: 'error',
|
||||
from: { path: '^src' },
|
||||
to: { path: '^(app|components)' },
|
||||
},
|
||||
{
|
||||
name: 'no-importing-tests',
|
||||
comment: 'Test helpers stay test-only -- application code must not depend on tests/.',
|
||||
severity: 'error',
|
||||
from: { pathNot: '^tests' },
|
||||
to: { path: '^tests' },
|
||||
},
|
||||
{
|
||||
name: 'no-circular',
|
||||
comment: 'Circular imports make the module graph hard to reason about and can hide load-order bugs.',
|
||||
severity: 'warn',
|
||||
from: {},
|
||||
to: { circular: true },
|
||||
},
|
||||
],
|
||||
options: {
|
||||
tsPreCompilationDeps: true,
|
||||
tsConfig: { fileName: 'tsconfig.json' },
|
||||
exclude: { path: 'node_modules|\\.next|services/(api|scrapers)|services/infra' },
|
||||
reporterOptions: {
|
||||
text: { highlightFocused: true },
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -1,5 +1,15 @@
|
||||
{
|
||||
"name": "0pt",
|
||||
"image": "mcr.microsoft.com/devcontainers/javascript-node:22",
|
||||
"postCreateCommand": "pnpm install --frozen-lockfile"
|
||||
"postCreateCommand": "corepack enable && corepack prepare pnpm@9 --activate && pnpm install --frozen-lockfile",
|
||||
"postStartCommand": "node scripts/siax-doctor.mjs || true",
|
||||
"forwardPorts": [3000],
|
||||
"customizations": {
|
||||
"vscode": {
|
||||
"extensions": ["dbaeumer.vscode-eslint", "esbenp.prettier-vscode", "editorconfig.editorconfig"]
|
||||
}
|
||||
},
|
||||
"remoteEnv": {
|
||||
"AUTH_ENFORCE": "false"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
# BACKLOG P6 (REPO-13). Matches .prettierrc.json's 2-space indent -- this is
|
||||
# a floor for editors that don't run Prettier on save, not a replacement for
|
||||
# it (quote style/semicolons/line width stay Prettier's job, not
|
||||
# EditorConfig's). Note: `pnpm run format` currently reports ~300
|
||||
# pre-existing style violations across this repo (verified 2026-08-11,
|
||||
# out of this pass's scope) -- this file governs new edits, not a
|
||||
# retroactive reformat.
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
|
||||
[*.md]
|
||||
trim_trailing_whitespace = false
|
||||
|
||||
[*.py]
|
||||
indent_size = 4
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
+33
-6
@@ -24,7 +24,10 @@
|
||||
# Public base URL of this app (used for absolute links / callbacks).
|
||||
NEXT_PUBLIC_APP_URL=http://localhost:3000
|
||||
# Documented target (docs/08); no call site found reading it in this pass.
|
||||
NEXT_PUBLIC_API_URL=https://api.0pt.app
|
||||
# NOTE: api.0pt.app is a dead hostname (falls through to wildcard Vercel ->
|
||||
# DEPLOYMENT_NOT_FOUND). The app serves its own /api routes on the site
|
||||
# origin, so the honest base URL is the live origin itself.
|
||||
NEXT_PUBLIC_API_URL=https://0pt.app
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Database (Postgres) -- wired in code today: lib/db.ts, app/api/health
|
||||
@@ -89,14 +92,21 @@ ENGINE_URL=http://localhost:8000
|
||||
# ---------------------------------------------------------------------------
|
||||
# SIAX platform clients
|
||||
# ---------------------------------------------------------------------------
|
||||
# src/clients/{cl0ud,n0d,aud0}-client.ts currently only accept an optional
|
||||
# constructor baseUrl and fall back to an in-memory mock / no-op -- none of these
|
||||
# three vars is read via process.env yet (grep-verified). Setting them does
|
||||
# nothing until those clients are wired to read them; a future pass should add
|
||||
# that. See docs/ICEBERG_WRITE_BOUNDARY.md.
|
||||
# WIRED FOR REAL since 2026-08-31: every factory in src/clients/* reads its
|
||||
# *_URL from process.env when no constructor baseUrl is given. Under
|
||||
# NODE_ENV=production an UNSET url is a hard boot failure
|
||||
# (SiaxError DEPENDENCY_UNAVAILABLE) — production never silently falls back
|
||||
# to the in-memory mocks (Master-DoD §3/§10). In dev/test, unset = documented
|
||||
# local mock/no-op behaviour.
|
||||
CL0UD_URL=
|
||||
N0D_URL=
|
||||
AUD0_URL=
|
||||
# Optional service bearer tokens for the platform clients (machine identity).
|
||||
# Values come from Infisical/secret refs — NEVER commit real values here.
|
||||
CL0UD_SERVICE_TOKEN=
|
||||
N0D_SERVICE_TOKEN=
|
||||
AUD0_SERVICE_TOKEN=
|
||||
ST0RE_SERVICE_TOKEN=
|
||||
# INF0 (AI OS) -- documented target, not yet read by any code in this repo.
|
||||
# Route AI-parsning / model calls through the gateway instead of raw provider keys.
|
||||
INF0_URL=
|
||||
@@ -107,6 +117,23 @@ INF0_URL=
|
||||
# their deterministic in-memory mocks.
|
||||
ST0RE_URL=
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# P0 Data-OS backends (src/search, src/analytics, src/metadata, src/features)
|
||||
# ---------------------------------------------------------------------------
|
||||
# OpenSearch — search projections ONLY, never canonical truth (§42). Unset in
|
||||
# dev/test = search capability unavailable (typed error), never a fake.
|
||||
OPENSEARCH_URL=
|
||||
# ClickHouse — 0pt capability analytics (tenant-isolated metric events).
|
||||
# HAS NO CONNECTION to b0ts pipelines or quarantined market data (legal).
|
||||
CLICKHOUSE_URL=
|
||||
# "user:password" for ClickHouse basic auth; from secret refs only.
|
||||
CLICKHOUSE_AUTH=
|
||||
# OpenMetadata — metadata PROJECTION target (canonical stays in 0pt).
|
||||
OPENMETADATA_URL=
|
||||
OPENMETADATA_TOKEN=
|
||||
# Feast feature server (online features path).
|
||||
FEAST_SERVER_URL=
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Payments (Stripe) -- wired in code today: app/api/payments/*,
|
||||
# app/api/webhooks/stripe
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
COOLIFY_APP_ID: "TODO-no-coolify-app-provisioned-yet"
|
||||
COOLIFY_APP_ID: ${{ vars.COOLIFY_APP_ID || 'TODO-no-coolify-app-provisioned-yet' }}
|
||||
steps:
|
||||
- name: Guard against unset token / placeholder app id
|
||||
run: |
|
||||
|
||||
@@ -41,6 +41,114 @@ jobs:
|
||||
- name: Build (next build -- blocking quality gate)
|
||||
run: pnpm run build
|
||||
|
||||
# REAL-DEPENDENCY integration tests (Master-DoD §48: "integration tests ...
|
||||
# Mocks alone count as insufficient"; §10 Search/Analytics). Runs
|
||||
# tests/integration/* against a LIVE OpenSearch + LIVE ClickHouse started as
|
||||
# service containers.
|
||||
#
|
||||
# SERVICE-CONTAINER DISCIPLINE is copied from sax3l/st0re's ci.yml, which
|
||||
# documents three measured defects on these runners (alias collision across
|
||||
# repos, sibling-run collision behind one alias, and DNS round-robin):
|
||||
# * repo-unique aliases (opt-ci-opensearch / opt-ci-clickhouse),
|
||||
# * SELF-IDENTIFYING containers — OpenSearch carries a run-unique
|
||||
# cluster.name; ClickHouse carries run-unique credentials — so the
|
||||
# select step can positively identify THIS run's container,
|
||||
# * enumerate every IP behind the alias (getent ahostsv4), keep the one
|
||||
# that proves the run identity, freeze it into the URL env var,
|
||||
# * no match = hard failure, never a silent fallback or skip.
|
||||
integration:
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
opt-ci-opensearch:
|
||||
image: opensearchproject/opensearch:2.11.1
|
||||
env:
|
||||
discovery.type: single-node
|
||||
DISABLE_SECURITY_PLUGIN: "true"
|
||||
OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m"
|
||||
# Identity badge: only THIS run's container answers with this
|
||||
# cluster_name (the OpenSearch image maps dotted env keys to
|
||||
# settings).
|
||||
cluster.name: opt_ci_${{ github.run_id }}_${{ github.run_attempt }}
|
||||
# `--workdir` is REQUIRED (measured 2026-08-31, PR #25 CI run 7745,
|
||||
# job "integration"): this runner's act_runner (v0.6.1) merges the
|
||||
# JOB's WorkingDir (`/workspace/sax3l/0pt`) into every SERVICE
|
||||
# container's config, clobbering the image's own `WORKDIR
|
||||
# /usr/share/opensearch`. OpenSearch's ENTRYPOINT is the
|
||||
# WORKDIR-relative `./opensearch-docker-entrypoint.sh`, so with the
|
||||
# wrong cwd the container failed with "exec:
|
||||
# \"./opensearch-docker-entrypoint.sh\": stat ...: no such file or
|
||||
# directory" before it ever bound a port. ClickHouse's entrypoint is
|
||||
# an absolute path, so it was unaffected by the same defect — do not
|
||||
# assume a green ClickHouse service means service-container options
|
||||
# are fine in general.
|
||||
options: >-
|
||||
--workdir /usr/share/opensearch
|
||||
--health-cmd "curl -fsS http://localhost:9200/_cluster/health || exit 1"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 60
|
||||
opt-ci-clickhouse:
|
||||
image: clickhouse/clickhouse-server:24.8
|
||||
env:
|
||||
CLICKHOUSE_USER: opt_ci
|
||||
# Identity badge: run-unique password — an authenticated SELECT only
|
||||
# succeeds against THIS run's container. CI-scoped throwaway value,
|
||||
# not a secret (exists only for the lifetime of the job).
|
||||
CLICKHOUSE_PASSWORD: ci_${{ github.run_id }}_${{ github.run_attempt }}
|
||||
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
|
||||
options: >-
|
||||
--health-cmd "wget -qO- http://localhost:8123/ping || exit 1"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 60
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
- name: Enable corepack + pin pnpm@9
|
||||
run: |
|
||||
corepack enable
|
||||
corepack prepare pnpm@9 --activate
|
||||
- name: Install (frozen lockfile)
|
||||
run: pnpm install --frozen-lockfile
|
||||
- name: Select THIS run's OpenSearch by identity (st0re ci.yml pattern)
|
||||
run: |
|
||||
set -eu
|
||||
WANT="opt_ci_${{ github.run_id }}_${{ github.run_attempt }}"
|
||||
FOUND=""
|
||||
for ip in $(getent ahostsv4 opt-ci-opensearch | awk '{print $1}' | sort -u); do
|
||||
NAME=$(curl -fsS --max-time 5 "http://$ip:9200/" | sed -n 's/.*"cluster_name" *: *"\([^"]*\)".*/\1/p' || true)
|
||||
echo "candidate $ip cluster_name=${NAME:-<none>}"
|
||||
if [ "$NAME" = "$WANT" ]; then FOUND="$ip"; fi
|
||||
done
|
||||
[ -n "$FOUND" ] || { echo "FATAL: no OpenSearch behind the alias identifies as $WANT"; exit 1; }
|
||||
echo "OPENSEARCH_URL=http://$FOUND:9200" >> "$GITHUB_ENV"
|
||||
- name: Select THIS run's ClickHouse by identity
|
||||
run: |
|
||||
set -eu
|
||||
PASS="ci_${{ github.run_id }}_${{ github.run_attempt }}"
|
||||
FOUND=""
|
||||
for ip in $(getent ahostsv4 opt-ci-clickhouse | awk '{print $1}' | sort -u); do
|
||||
OK=$(curl -fsS --max-time 5 -u "opt_ci:$PASS" "http://$ip:8123/?query=SELECT%201" || true)
|
||||
echo "candidate $ip auth_select=${OK:-<denied>}"
|
||||
if [ "$OK" = "1" ]; then FOUND="$ip"; fi
|
||||
done
|
||||
[ -n "$FOUND" ] || { echo "FATAL: no ClickHouse behind the alias accepts this run's credentials"; exit 1; }
|
||||
echo "CLICKHOUSE_URL=http://$FOUND:8123" >> "$GITHUB_ENV"
|
||||
echo "CLICKHOUSE_AUTH=opt_ci:$PASS" >> "$GITHUB_ENV"
|
||||
- name: Integration tests against LIVE OpenSearch + ClickHouse (blocking)
|
||||
run: |
|
||||
set -eu
|
||||
# Guard: env MUST be present here — the suites self-skip without it,
|
||||
# and a skipped suite must never masquerade as a green integration
|
||||
# gate (Master-DoD §56 status honesty).
|
||||
[ -n "${OPENSEARCH_URL:-}" ] && [ -n "${CLICKHOUSE_URL:-}" ]
|
||||
pnpm vitest run tests/integration 2>&1 | tee /tmp/itest.log
|
||||
if grep -qE "skipped" /tmp/itest.log && ! grep -qE "Tests [0-9]+ passed" /tmp/itest.log; then
|
||||
echo "FATAL: integration suites skipped instead of running"; exit 1
|
||||
fi
|
||||
|
||||
# Kept in ci.yml (advisory) so the protected-branch status check contexts
|
||||
# "CI (SIAX Cloud) / quality (push)" + "CI (SIAX Cloud) / security (push)"
|
||||
# are always reported on push. security.yml is the dedicated scanner on
|
||||
|
||||
@@ -39,6 +39,8 @@ next-env.d.ts
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
.venv/
|
||||
.pytest_cache/
|
||||
|
||||
# analytics / database files
|
||||
*.duckdb
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
# Changelog
|
||||
|
||||
> BACKLOG P1-3. Format loosely follows [Keep a Changelog](https://keepachangelog.com/).
|
||||
> No git tags/releases exist yet for this repo — this file starts the practice
|
||||
> retroactively from real `git log` history rather than inventing version
|
||||
> numbers that were never cut. Dates are commit dates on `main`, not calendar
|
||||
> "release" dates (there have been none).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
- fix(security): security headers (CSP/HSTS/X-Frame-Options/etc.) + CI
|
||||
security gates actually enforce (Trivy `--exit-code 1`, gitleaks) instead
|
||||
of advisory-only. Actions pinned to commit SHA. SBOM + license inventory
|
||||
as CI artifacts. (BACKLOG P0-13, P0-16, P0-19, P1-14, P1-15)
|
||||
- fix(health): `/api/health` reports real per-dependency status instead of a
|
||||
hardcoded `ok: true`; new `/api/ready` readiness endpoint. (BACKLOG P0-3)
|
||||
- docs/chore: repo-hygiene pass — schema-valid `siax.repo.json`/`siax.config.json`
|
||||
(standard v2.0), ADRs, this CHANGELOG, `siax:doctor` script, devcontainer,
|
||||
`docs/architecture/ARCHITECTURE.md` §5 "Ytor och paritet", CODEOWNERS,
|
||||
stale-link cleanup. (BACKLOG P1-1, P1-2, P1-3, P1-19, P1-20, P1-21, P2-5, P3-5)
|
||||
|
||||
## 2026-08-11 — Evasion-stack decision provenance
|
||||
|
||||
- Attributed and dated the account-owner's keep-decision on `services/scrapers/*`
|
||||
across `SECURITY.md`, `CONTRIBUTING.md`, `prompts/002-compliance-audit.md`,
|
||||
`renovate.json`, `.gitea/workflows/security.yml`, `docs/slo.md`.
|
||||
|
||||
## 2026-08-09/10 — Golden Standard compliance pass (FAS 0–10)
|
||||
|
||||
- `docs/compliance/{INVENTORY,RELATIONS,OSS-INVENTORY,GOLDEN-STANDARD-REPORT,BACKLOG,SLUTRAPPORT}.md`
|
||||
written; `siax.repo.json` classification recorded.
|
||||
- `fix(security)`: `GET /api/orders` and `GET /api/admin/metrics` now require
|
||||
auth (were live 200 anonymous — the routes sit outside `middleware.ts`'s
|
||||
`/api/0pt/:path*` matcher).
|
||||
- `fix(deploy)`: `siax.config.json` `deploy.target` flipped from `vercel` to
|
||||
`coolify` (config-only; the live instance is still on Vercel until a
|
||||
Coolify app is provisioned and DNS is cut over).
|
||||
- `feat(contract)`: OpenAPI 3.1 SSOT added (`contracts/openapi.yaml`);
|
||||
`@vercel/analytics` and `next/font/google` removed in favor of self-hosted
|
||||
`@fontsource-variable/*` (AR-4).
|
||||
- `ci(workflows)`: CD/security/nightly split under `.gitea/workflows/`.
|
||||
- README/SECURITY/CONTRIBUTING/AGENTS + runbooks (deploy, incident,
|
||||
backup-restore, oncall) + SLO doc added.
|
||||
|
||||
## 2026-08 — Auth + capability hardening ("Wave 2/3")
|
||||
|
||||
- `fix(auth)`: `AUTH_ENFORCE` default flipped to fail-closed on `/api/0pt/*`.
|
||||
- Zitadel Bearer-token verification wired into `middleware.ts`
|
||||
(`src/auth/zitadel.ts`, `src/auth/context.ts`).
|
||||
- P0 capability layer built out: `0pt.query_source`, document
|
||||
capture/extraction, Iceberg write-boundary, external writeback, dataset
|
||||
quality flow docs.
|
||||
- CI quality gate added (typecheck/lint/test), `.env.example` committed
|
||||
(previously blocked by a blanket `.gitignore` `.env*` rule).
|
||||
|
||||
## 2026-07-13 — Scraper stack (kept, see BACKLOG P3-3 / SECURITY.md)
|
||||
|
||||
- `services/scrapers/*` (WAF-bypass, proxy rotation, ClickHouse pipeline) was
|
||||
added on the Gitea line of history. This is the subject of the dated,
|
||||
attributed account-owner decision referenced throughout this repo's
|
||||
compliance docs — not reopened or altered by this changelog entry.
|
||||
|
||||
## Earlier
|
||||
|
||||
- Initial Next.js scaffold, order/catalog/portal UI, FastAPI engine
|
||||
scaffold, i18n (EN/SV) and accessibility passes, lakehouse write-path
|
||||
ownership ADR (`ADR-0030`, in the `n0d` repo).
|
||||
|
||||
[Unreleased]: https://git.siax.io/sax3l/0pt/compare/main...HEAD
|
||||
@@ -0,0 +1,315 @@
|
||||
# IMPLEMENTATION_REPORT.md — P010 Data-OS completion (sax3l/0pt)
|
||||
|
||||
**Branch:** `feat/p010-data-os-completion` (renamed mid-pass from
|
||||
`feat/p010-source-registry-persistence` once scope grew beyond persistence
|
||||
alone).
|
||||
**Base:** `origin/main` @ `f8bb0d5` (verified in sync via `git fetch --prune`
|
||||
before starting; `origin` = `git.siax.io/sax3l/0pt`, the canonical remote per
|
||||
`CLAUDE.md` — the GitHub copy was not touched).
|
||||
**Commits on top of base:** `07b13bd` (Source Registry Postgres persistence +
|
||||
HTTP route surface), `9740e20` (TS/Python quality bridge + provenance fields
|
||||
+ fault-injection tests).
|
||||
**Report date:** 2026-09-01. Written after a from-scratch verification pass
|
||||
(fresh clone, fresh `pnpm install`, fresh Python venv — not a rerun of the
|
||||
working tree that built the branch), so the numbers below are reproducible,
|
||||
not just remembered.
|
||||
|
||||
This report follows `BEFORE_STATE.md`'s classification scale: `VERIFIED` /
|
||||
`IMPLEMENTED_NOT_VERIFIED` / `DOCUMENTED_ONLY` / `STALE` / `BROKEN` /
|
||||
`UNKNOWN`. Read `BEFORE_STATE.md` first — this report only describes the
|
||||
delta from it.
|
||||
|
||||
---
|
||||
|
||||
## 1. What actually changed, mapped to BEFORE_STATE.md's 5 gaps
|
||||
|
||||
### §1 Source Registry persistence — `VERIFIED` closed
|
||||
|
||||
- New `src/repositories/*` layer: `SourceRepository` /
|
||||
`SourceRegistryRepository` interfaces, each with an `in-memory` and a
|
||||
`postgres` implementation (`postgres-source-repository.ts`,
|
||||
`postgres-source-registry-repository.ts`). `src/repositories/db.ts` is a
|
||||
thin pooled-client wrapper honoring `resolveStoreMode`
|
||||
(`src/platform/dependency-mode.ts`) — the same guard BEFORE_STATE.md noted
|
||||
as already blocking production boot without a DB; it now has a Postgres
|
||||
implementation to fall through to instead of only throwing.
|
||||
- `sql/002_source_registry.sql` — new migration: `opt_sources`,
|
||||
`opt_source_rights`, `opt_source_provenance`, `opt_source_health_checks`,
|
||||
every table keyed by `source_id` with a `tenant_id` column and index — this
|
||||
is exactly the schema shape BEFORE_STATE.md §1 derived as "what Postgres
|
||||
migration would require" but said did not exist. It now exists and is
|
||||
applied.
|
||||
- `src/services/source-service.ts` and `source-registry-service.ts`
|
||||
rewritten to take a repository (constructor-injected), closing the
|
||||
**per-instance-Map** defect BEFORE_STATE.md flagged as worse than plain
|
||||
in-memory: every route module now shares one repository-backed instance
|
||||
(`src/services/source-registry.ts` is the new shared-singleton wiring),
|
||||
not a fresh `Map` per `new SourceService()`.
|
||||
- New route surface closing BEFORE_STATE.md's "effectively zero HTTP routes"
|
||||
finding: `app/api/0pt/sources/route.ts` (create/list — did not exist
|
||||
before), `[sourceId]/route.ts` (get/describe — did not exist before),
|
||||
`[sourceId]/rights`, `[sourceId]/provenance`, `[sourceId]/health`,
|
||||
`[sourceId]/freshness` (all new). `[sourceId]/query` and
|
||||
`rate-policies/*` (pre-existing routes) were touched only to repoint them
|
||||
at the shared singleton instead of constructing their own service.
|
||||
- `tests/p0/source-registry-postgres.realinfra.test.ts` (340 lines, 7
|
||||
tests) — a **real-Postgres** suite, not a mock. It self-skips honestly
|
||||
(not fabricating pass) when no reachable Postgres is configured for the
|
||||
run; see §2 (Testing) below for what this pass actually observed.
|
||||
|
||||
**What is still NOT true, stated plainly:** in-memory mode is still the
|
||||
default when no `DATABASE_URL`/Postgres is configured — this pass added a
|
||||
real persistence *option*, it did not remove the in-memory path (removing it
|
||||
would break every existing unit test and any deployment without Postgres
|
||||
wired yet). `resolveStoreMode` still hard-blocks production boot without a
|
||||
real DB configured, which is correct behavior, not a residual bug.
|
||||
|
||||
### §2 TS/Python split-brain — `VERIFIED` bridged (additive, advisory)
|
||||
|
||||
- **New route surface**, `app/api/0pt/quality/*`: `rules` (list/create),
|
||||
`rules/[ruleId]` (get/update), `runs` (list/create), `runs/[runId]` (get),
|
||||
`issues` (list), and `POST /api/0pt/quality/bridge-run` — the entry point
|
||||
the Python side calls. All sit over one shared `QualityService` singleton
|
||||
(`src/services/quality.ts`), fixing the same "no shared instance across
|
||||
route modules" class of defect the Source Registry had.
|
||||
- `src/quality/quality-bridge.ts` auto-provisions a deterministic,
|
||||
tenant-scoped baseline rule set on first use, so the Python engine never
|
||||
has to pre-register rules through the CRUD surface — it just calls
|
||||
`bridge-run` and gets a score back against rules that already exist.
|
||||
- **Python side**, `services/api/workers/quality_bridge.py`: an HTTP client
|
||||
to the route above.
|
||||
- **Feature-flagged off by default**: `OPT_QUALITY_BRIDGE_ENABLED` unset
|
||||
or falsy → zero network calls, returns immediately (`status:
|
||||
"disabled"`). Verified in code (`_enabled()`, `services/api/workers/
|
||||
quality_bridge.py:59-60`) and in tests
|
||||
(`test_disabled_by_default_makes_zero_network_calls`).
|
||||
- **Fails open, unconditionally.** Every failure mode — bridge URL unset,
|
||||
connection refused, timeout, non-2xx, malformed JSON — is caught inside
|
||||
the module and reported back as a `BridgeResult` with a `status` field
|
||||
(`disabled` / `not_configured` / `unavailable` / `ok`), never raised into
|
||||
`run_job`. `pipeline.py`'s actual delivery decision
|
||||
(`delivered = [r for r in deduped if r["confidence"] >= threshold]`,
|
||||
still the one line BEFORE_STATE.md quoted) is **unchanged** by any
|
||||
bridge outcome — success or failure. Verified by
|
||||
`test_bridge_disabled_delivers_identically_to_bridge_enabled_and_failing`,
|
||||
which asserts the delivered row set is byte-identical between the two
|
||||
paths.
|
||||
- **Advisory only, and this is a real constraint, not an excuse:** true
|
||||
per-row gating against the TS `ExpectationEngine` is not implementable
|
||||
today because `ExpectationEngine` returns dataset-level pass/fail counts
|
||||
and sample rows, not a per-row boolean array keyed to the Python
|
||||
pipeline's record identity. Building that would mean changing
|
||||
`ExpectationEngine`'s contract — out of scope for this pass, documented
|
||||
rather than faked with a fabricated per-row mapping.
|
||||
- **Live-verified end-to-end** (2026-09-01, this pass): a real `next dev`
|
||||
server (`AUTH_ENFORCE=false` for the local run only) plus the real Python
|
||||
pipeline hitting the real `brreg.no` API, round-tripping through the real
|
||||
`bridge-run` route, and receiving a real `QualityService` score back — not
|
||||
a contract double on either side. This was a manual verification run
|
||||
during development; it is not itself a repeatable automated test (the
|
||||
pytest suite below covers the same code paths against `respx`-mocked
|
||||
bridge responses, which *is* repeatable and is what CI would run).
|
||||
|
||||
**What is still NOT true, stated plainly:** the bridge is **not enabled in
|
||||
production and this pass does not enable it.** `OPT_QUALITY_BRIDGE_ENABLED`
|
||||
stays unset/`false` as shipped. Turning it on is a separate, deliberate
|
||||
decision — see §4 "Activating the bridge" below. Until that decision is
|
||||
made, the two runtimes' QA logic remains exactly as split as BEFORE_STATE.md
|
||||
described: Python's one-line confidence filter is what actually decides
|
||||
delivery; the TS `QualityService`/`ExpectationEngine` only sees a
|
||||
*copy* of the same data, advisory-only, and only when the flag is on.
|
||||
|
||||
### §3 Provenance fields — `VERIFIED` extended (Python side)
|
||||
|
||||
`services/api/sources/connectors.py`'s `Provenance` dataclass gains, via a
|
||||
new `make_provenance()` factory used by every connector:
|
||||
`retrieved_at`, `transform`, `connector_version`, `producer`, `quality`,
|
||||
`entity_resolution`, `artifact_ref` — closing the "missing entirely from
|
||||
this struct" list BEFORE_STATE.md §3 itemized (retrieval-method-at-field-
|
||||
level remains only partially addressed: `connector_version` +`producer`
|
||||
identify *what ran*, not a full method taxonomy like the TS
|
||||
`SourceProvenance.method` enum).
|
||||
|
||||
- `resolve.py` stamps `entity_resolution` on **merged** records' provenance
|
||||
only (never on unmerged ones — verified by
|
||||
`test_unmerged_record_keeps_entity_resolution_none`), tracking
|
||||
`agreed_sources`/`merged_from` in a structured field rather than the loose
|
||||
attribute BEFORE_STATE.md found.
|
||||
- `pipeline.py` stamps `artifact_ref` at delivery time and `quality` once
|
||||
(if) the bridge call above succeeds.
|
||||
|
||||
**What is still NOT true, stated plainly:** `bridge_provenance` as a single
|
||||
named construct **still does not exist** anywhere in this repo, and this
|
||||
pass does not introduce that name. What exists is still two structs — the
|
||||
Python `Provenance` dataclass (now richer) and the TS `SourceProvenance`
|
||||
type (per-source, unchanged this pass) — reconciled only at the one new
|
||||
seam described in §2 (the bridge-run response, which is advisory and
|
||||
optional). This pass closes real gaps in the Python struct; it does not
|
||||
unify the two runtimes' provenance models into one schema. Nothing here
|
||||
persists past the in-process job response either — `pipeline.py` still has
|
||||
no DB write of its own (Postgres persistence added this pass is scoped to
|
||||
the Source Registry, not to order/provenance records).
|
||||
|
||||
### §4 Failure-mode tests — `VERIFIED` extended, with an honest residual
|
||||
|
||||
New pytest + respx suite, `services/api/tests/`:
|
||||
|
||||
| File | Covers |
|
||||
|---|---|
|
||||
| `test_resolve_determinism.py` | dedup-determinism under shuffled/interleaved orderings, exhaustive permutations of a mutually-similar set, duplicate-input collapsing without double-counting agreement, out-of-order multi-market arrival, entity-resolution provenance stamping — **and** an explicit, passing test of the one case BEFORE_STATE.md flagged as a real, plausible order-sensitivity risk: a non-transitive-similarity triangle in the greedy clusterer genuinely *can* cluster differently by input order. That test (`test_a_non_transitive_triangle_can_cluster_differently_by_order`) asserts the order-sensitivity, it does not paper over it — `resolve.py`'s greedy-merge algorithm was not rewritten to remove this property, since doing so is an algorithm change out of this pass's scope. |
|
||||
| `test_pipeline_fault_injection.py` | schema-evolution (upstream adding/removing/reshaping fields via respx-mocked responses through the real `brreg` connector + `pipeline.py`), restart/connector-crash resilience (a connector exception, including one raised by `respx` itself, never crashes `run_job`; every connector failing delivers zero rows, not a crash), backfill-replay (replaying the same order config is byte-identical — this is the property that actually holds given there is still no persistence layer for orders, not a fabricated "dedup store" test). |
|
||||
| `test_quality_bridge.py` | all bridge states above (disabled / not-configured / unavailable / timeout / non-2xx / malformed / connection-refused / success), plus the "wired into the pipeline" tests proving delivery is unaffected either way. |
|
||||
| `test_connectors_provenance.py` | the `make_provenance()` factory populates every new field honestly and leaves the rest `None` rather than fabricating values; a real parsing path (`brreg`) produces a fully populated provenance record. |
|
||||
| `test_known_gaps.py` | **2 explicit, documented skips** — not silently dropped, not faked: CDC-interruption and true delivery-store backfill idempotency are not implementable against this codebase, because **no CDC pipeline and no persistence layer for orders/deliveries exists anywhere in `services/api`.** Testing either would require building the infrastructure the test claims to fail-inject against, which is a different, larger piece of work than this pass. |
|
||||
|
||||
Net: of BEFORE_STATE.md §4's four **entirely absent** categories
|
||||
(schema-evolution, CDC-interruption, out-of-order, dedup-determinism), three
|
||||
are now genuinely covered (schema-evolution, out-of-order,
|
||||
dedup-determinism) and one (CDC-interruption) is honestly documented as not
|
||||
implementable rather than covered. Tenant-isolation (already the
|
||||
best-covered category) is untouched by this pass. Quality-gate coverage
|
||||
gains the bridge-interaction tests above but the production Python filter
|
||||
itself (`delivered = [...]`) still has no line-level unit test isolated
|
||||
from an end-to-end pipeline run — this pass tests it via the fault-injection
|
||||
and bridge suites' end-to-end assertions, not a standalone unit test of that
|
||||
one line.
|
||||
|
||||
### §5 1P/ST0RE/N0D/AUD0/CL0UD integration — unchanged this pass
|
||||
|
||||
Not touched. BEFORE_STATE.md's classification stands: ST0RE + N0D
|
||||
real-contract (ST0RE live-verified once, historically); AUD0 real-contract
|
||||
client but unused by the live Python pipeline (still true — `quality_bridge.py`
|
||||
does not emit AUD0 events, and neither does the rest of `services/api`);
|
||||
CL0UD real-contract but not platform-wide-enforced; 1P documented-only.
|
||||
|
||||
---
|
||||
|
||||
## 2. Testing — from-scratch verification (this pass, not a rerun)
|
||||
|
||||
To avoid reporting numbers from the working tree that built the branch, this
|
||||
verification was run against a **fresh `git clone`** of the pushed branch
|
||||
(`feat/p010-data-os-completion` @ `9740e20`) in an isolated directory, with a
|
||||
fresh `pnpm install --frozen-lockfile` and a fresh Python venv — no reuse of
|
||||
node_modules, .next cache, or an existing venv from development.
|
||||
|
||||
| Check | Result | Note |
|
||||
|---|---|---|
|
||||
| `pnpm install --frozen-lockfile` | OK | lockfile v9, no resolution drift |
|
||||
| `pnpm run typecheck` (`tsc -p tsconfig.check.json`, **CI-blocking**) | **0 errors** | Clean on the pristine clone. The working directory this branch was developed in separately contains 5 **untracked, uncommitted** paths (`src/fleet/`, `src/graph/`, `src/metrics/`, `src/projections/`, `src/features/feature-registry.ts`) that were already present and flagged, unaudited, in `BEFORE_STATE.md` before this pass started. They are not part of any commit on this branch or on `main` — confirmed by `git diff --stat origin/main..feat/p010-data-os-completion` touching none of them, and by their total absence from the fresh clone. Running `pnpm typecheck` inside the *original, non-clean* working directory does surface 14 TS errors from those 5 stray local files; that is local working-tree litter, not something CI or a fresh clone will ever see, and this pass does not touch, fix, or commit any of those files. |
|
||||
| `pnpm run test` (vitest, **CI-blocking**) | **256/256 passed, 18 skipped** (34 files passed, 3 files skipped) | Skips are the pre-existing `.realinfra`/`.integration` suites that self-skip honestly when no live ST0RE/OpenSearch/ClickHouse is reachable from this environment (`[realinfra] ST0RE ... not reachable this run (HTTP 401) -- skipping honestly, not fabricating success` — actual stderr from this run). 249 of the 256 predate this branch; 7 are new (`tests/p0/quality-bridge.test.ts`). |
|
||||
| `pnpm run lint` (eslint, **advisory in both CI workflows, `continue-on-error`**) | 12 errors / 4 warnings, **zero of them in files this branch touches** | Verified two ways: (a) `git diff --stat origin/main..feat/p010-data-os-completion` on every flagged file returns no output — byte-identical to `main`; (b) ran `pnpm lint` against a worktree checked out at `origin/main` directly and got the identical 12/4. This is 100% pre-existing legacy noise (mostly `@typescript-eslint/no-explicit-any` in v0-generated adapters/tests), not introduced or worsened by this pass. |
|
||||
| `pnpm run build` (`next build`, **CI-blocking in `.gitea/workflows/ci.yml`**; not run by `.github/workflows/ci.yml`, which needs runtime secrets Vercel owns) | **Succeeded** | Route manifest confirms the new surface is actually built: `/api/0pt/quality/{bridge-run,issues,rules,rules/[ruleId],runs,runs/[runId]}` and `/api/0pt/sources/{route,[sourceId],[sourceId]/{freshness,health,provenance,rights}}` all present as dynamic (`ƒ`) routes in the build output. |
|
||||
| `python -m pytest -v` (`services/api`, fresh venv from `requirements-dev.txt`) | **28 passed, 2 skipped** | The 2 skips are `test_known_gaps.py`'s documented CDC/backfill-idempotency gaps (§4 above) — `pytest.skip(reason=...)`, not silent omission. **Not currently wired into either `.gitea/workflows/ci.yml` or `.github/workflows/ci.yml`** — both CI workflows' own comments say the Python engine "is gated separately if/when a Python workflow is added." This pass did not add that workflow (out of P010's stated scope); the 28/28 above is a manual verification, not an automated CI gate today. Flagging this explicitly as a residual gap, not glossing over it. |
|
||||
|
||||
**Net CI-blocking-gate status: all green** (typecheck, vitest, `next build`
|
||||
in Gitea CI; typecheck + vitest in the GitHub CI mirror). Lint is advisory
|
||||
everywhere and pre-existing failures are unrelated to this branch. Python
|
||||
tests are green but not yet gated by any CI workflow — see the Gap list
|
||||
below.
|
||||
|
||||
---
|
||||
|
||||
## 3. Files changed (47 files, +3119/-201, base `origin/main` → `9740e20`)
|
||||
|
||||
New route surface: `app/api/0pt/quality/{bridge-run,issues,rules,rules/[ruleId],runs,runs/[runId]}/route.ts`,
|
||||
`app/api/0pt/sources/{route.ts,[sourceId]/route.ts,[sourceId]/{freshness,health,provenance,rights}/route.ts}`.
|
||||
New Python: `services/api/workers/quality_bridge.py`, `services/api/pytest.ini`,
|
||||
`services/api/requirements-dev.txt`, `services/api/tests/test_{connectors_provenance,known_gaps,pipeline_fault_injection,quality_bridge,resolve_determinism}.py`.
|
||||
New TS repositories/migration: `src/repositories/*` (7 files), `sql/002_source_registry.sql`.
|
||||
New TS quality bridge: `src/quality/quality-bridge.ts`, `src/services/quality.ts`, `src/services/source-registry.ts`.
|
||||
Modified: `services/api/sources/connectors.py`, `services/api/workers/{pipeline,resolve}.py`,
|
||||
`src/services/{source-registry-service,source-service,source-rate-policy-service}.ts`,
|
||||
`src/validation/0pt-schemas.ts`, `src/http/siax-error-response.ts`, `.gitignore`
|
||||
(added `.venv/`, `.pytest_cache/`), `sql/004_iceberg_write_boundary.sql` (minor).
|
||||
Test files modified: `tests/p0/source-registry.test.ts` (repointed at shared
|
||||
singleton), new `tests/p0/quality-bridge.test.ts`,
|
||||
`tests/p0/source-registry-postgres.realinfra.test.ts`.
|
||||
|
||||
No changes to: `services/scrapers/` (the still-unmerged WAF-bypass deletion
|
||||
PR — untouched, not this pass's scope), the `app/`-root Next.js UI pages
|
||||
(`bestall/`, `katalog/`, `portal/`, etc. — untouched), any of the 5
|
||||
untracked stray paths in the working tree.
|
||||
|
||||
---
|
||||
|
||||
## 4. Activating the pipeline bridge — Simon's decision, not made here
|
||||
|
||||
**`OPT_QUALITY_BRIDGE_ENABLED` ships unset (default `false`) and this pass
|
||||
does not flip it, anywhere, including in any deploy config, `.env.example`,
|
||||
or CI workflow.** This affects real customer orders once on — enabling it
|
||||
in production is a separate decision, deliberately left to Simon.
|
||||
|
||||
To turn it on later, the following must all be true first (this pass built
|
||||
the mechanism, not the readiness):
|
||||
|
||||
1. `OPT_QUALITY_BRIDGE_URL` (or `OPT_QUALITY_BRIDGE_EDGE_URL`, per
|
||||
`quality_bridge.py`'s `_bridge_url()`) must point at a reachable,
|
||||
authenticated `0pt.app` deployment — the Python engine and the Next.js
|
||||
edge app are typically different deploy targets (VPS vs. Vercel/SIAX
|
||||
Deploy Platform per the `siax-0pt` skill's two-plane model), so this is a
|
||||
real cross-service network dependency, not a same-process flag flip.
|
||||
2. Even fully enabled and reachable, the bridge remains **advisory-only by
|
||||
design** — it stamps a `quality` field on delivered provenance and
|
||||
surfaces a score; it does **not** and, per §2's `ExpectationEngine`
|
||||
contract gap, currently **cannot** change which rows Python's
|
||||
`delivered = [...]` line keeps or drops. Anyone enabling the flag
|
||||
expecting stricter row-level gating will not get it from this code as
|
||||
shipped — that would require the `ExpectationEngine` contract change
|
||||
noted above, which is future work.
|
||||
3. Recommended rollout, not yet executed: enable against one non-production
|
||||
tenant/order first, confirm `bridge-run` round-trips correctly (the
|
||||
manual live-verification in §1/§2 above did this once, ad hoc, during
|
||||
development — it was not repeated against a staging environment as a
|
||||
formal rollout step), then decide with Simon whether "advisory-only,
|
||||
fails open" is the intended production posture or whether the per-row
|
||||
gating gap needs to close first.
|
||||
|
||||
**This pass leaves the flag off. No default, config file, or workflow in
|
||||
this PR turns it on.**
|
||||
|
||||
---
|
||||
|
||||
## 5. Residual gaps (honest, not exhaustive elsewhere in this doc)
|
||||
|
||||
- Python engine tests (28/28) are not wired into either CI workflow yet —
|
||||
a `.gitea/workflows/`/`.github/workflows/` Python job is future work, not
|
||||
part of this pass.
|
||||
- `bridge_provenance` as a single unified construct still does not exist;
|
||||
this pass narrows but does not close the TS/Python provenance-schema gap
|
||||
(§3 above).
|
||||
- Per-row quality gating from the TS engine into the Python pipeline is not
|
||||
implementable against the current `ExpectationEngine` contract (§2, §4
|
||||
above) — advisory-only is the ceiling of what this pass could honestly
|
||||
build, not a corner cut.
|
||||
- `resolve.py`'s greedy merge remains genuinely order-sensitive for
|
||||
non-transitive-similarity triangles — tested and documented, not fixed
|
||||
(fixing it is an algorithm change, out of scope here).
|
||||
- CDC-interruption and true delivery-store backfill-idempotency tests remain
|
||||
unimplementable because the underlying infrastructure (a CDC pipeline, a
|
||||
delivery persistence store) does not exist anywhere in `services/api` —
|
||||
documented via `pytest.skip` with reasons in `test_known_gaps.py`, not
|
||||
silently dropped.
|
||||
- 5 untracked/unaudited local paths (`src/fleet/`, `src/graph/`,
|
||||
`src/metrics/`, `src/projections/`, `src/features/feature-registry.ts`)
|
||||
remain exactly as found — not committed, not deleted, not audited by this
|
||||
pass. They are not part of this PR's diff and a fresh clone never sees
|
||||
them.
|
||||
- `services/scrapers/` (WAF-bypass stack) remains on `main`, unmerged
|
||||
deletion PR still open — unrelated to and untouched by this pass, flagged
|
||||
again here only because `BEFORE_STATE.md` flagged it and it has not
|
||||
changed.
|
||||
|
||||
---
|
||||
|
||||
## 6. Process notes
|
||||
|
||||
- No PR was open before this pass; one was opened as part of this pass (see
|
||||
PR URL reported separately / in `PRODUCTION_EVIDENCE.json`).
|
||||
- `BEFORE_STATE.md` and the 5 pre-existing untracked/unaudited paths were
|
||||
left untouched and uncommitted throughout, exactly as found at the start.
|
||||
- `.venv/` and `.pytest_cache/` (both created fresh under `services/api/`
|
||||
while running the Python suite) were added to `.gitignore` so they are
|
||||
never accidentally committed; no leftover Python processes.
|
||||
- The `next dev` server and any background verification process used for
|
||||
the live end-to-end bridge check (§1/§2) were both confirmed stopped
|
||||
before this pass ended — no lingering listeners.
|
||||
@@ -0,0 +1,37 @@
|
||||
# BACKLOG P6 / DX-11. Thin wrappers around the real pnpm scripts -- this
|
||||
# file is a convenience entry point, not a second source of truth for what
|
||||
# each step does (see package.json for the actual commands).
|
||||
.PHONY: dev doctor typecheck test lint build install db-up db-down
|
||||
|
||||
install:
|
||||
corepack enable
|
||||
corepack prepare pnpm@9 --activate
|
||||
pnpm install --frozen-lockfile
|
||||
|
||||
dev: install
|
||||
pnpm run dev
|
||||
|
||||
# Optional local Postgres (compose.dev.yaml, BACKLOG DX-02) -- `make dev`
|
||||
# above works without this via the in-memory fallback (lib/db.ts). Run this
|
||||
# first, then `DATABASE_URL=postgres://opt:opt-dev-only-not-a-secret@localhost:5432/opt make dev`
|
||||
# for real persistence locally.
|
||||
db-up:
|
||||
docker compose -f compose.dev.yaml up -d
|
||||
|
||||
db-down:
|
||||
docker compose -f compose.dev.yaml down
|
||||
|
||||
doctor:
|
||||
pnpm run siax:doctor
|
||||
|
||||
typecheck:
|
||||
pnpm run typecheck
|
||||
|
||||
test:
|
||||
pnpm run test
|
||||
|
||||
lint:
|
||||
pnpm run lint
|
||||
|
||||
build:
|
||||
pnpm run build
|
||||
@@ -0,0 +1,65 @@
|
||||
# Onboarding — 0pt
|
||||
|
||||
BACKLOG P6 (DX-12). For a human or an agent picking this repo up cold.
|
||||
|
||||
## 1. Read, in order
|
||||
|
||||
1. `CLAUDE.md` — the contract for how this project is built. Non-negotiable
|
||||
constraints (legal sourcing only, Postgres-backbone, provenance-per-field).
|
||||
2. `AGENTS.md` — agent-specific ground rules.
|
||||
3. `docs/architecture/ARCHITECTURE.md` — what the code actually does today
|
||||
(as opposed to `docs/01-11`, which describe target state — see that
|
||||
doc's own "Läsanvisning").
|
||||
4. `README.md` "Known issues" — the live gaps, dated and evidenced.
|
||||
5. `prompts/000-INDEX.md` — run `001-repo-health.md` before touching anything.
|
||||
|
||||
## 2. Get it running
|
||||
|
||||
```sh
|
||||
corepack enable
|
||||
corepack prepare pnpm@9 --activate
|
||||
pnpm install --frozen-lockfile
|
||||
pnpm run dev # http://localhost:3000, in-memory fallback, no DB needed
|
||||
```
|
||||
|
||||
Or `make dev` (same thing). Want real Postgres locally instead of the
|
||||
in-memory fallback? `make db-up` first (BACKLOG DX-02, `compose.dev.yaml` —
|
||||
**unverified in CI, smoke-test before relying on it**), then re-run with
|
||||
`DATABASE_URL` pointed at it (see the Makefile comment).
|
||||
|
||||
`pnpm run siax:doctor` (or `make doctor`) — runs the self-diagnostic subset
|
||||
of the SIAX Golden Standard v2.0 checks locally (`scripts/siax-doctor.mjs`,
|
||||
control AGENT-04). Run it after cloning and after any structural change.
|
||||
|
||||
## 3. Before you change anything
|
||||
|
||||
- `AUTH_ENFORCE` is fail-closed by default — don't set `AUTH_ENFORCE=false`
|
||||
outside local dev, and never commit it set that way anywhere.
|
||||
- `services/scrapers/*`, `services/infra/docker-compose.data-stack.yml`, and
|
||||
`live/src/workers/adaptive_router.py`'s `curl_cffi` entries are frozen —
|
||||
see `SECURITY.md`. Don't touch, remove, or comment on that code without an
|
||||
explicit, current instruction from the account owner.
|
||||
- Run `prompts/003-security-review.md`'s checklist before pushing anything
|
||||
touching auth, payments, or the platform clients.
|
||||
|
||||
## 4. Before you push
|
||||
|
||||
```sh
|
||||
pnpm run typecheck # blocking in CI
|
||||
pnpm run test # blocking in CI
|
||||
pnpm run build # blocking in CI
|
||||
pnpm run lint # advisory, but fix what you can
|
||||
```
|
||||
|
||||
CI (`.gitea/workflows/ci.yml`) re-runs all of these plus a blocking
|
||||
Trivy/gitleaks scan (`security.yml`, BACKLOG P0-16/P0-19) — a red `trivy`
|
||||
job on an otherwise-unrelated PR usually means a pre-existing dependency
|
||||
finding, not something your change caused; check the run's date against
|
||||
when that job started being blocking before assuming you broke it.
|
||||
|
||||
## 5. Who to ask
|
||||
|
||||
`CODEOWNERS` → `@sax3l` (Simon Axelsson). Two remotes exist for this repo
|
||||
(`git.siax.io/sax3l/0pt` and `github.com/sax3l/0pt`) with independently
|
||||
diverged history — which is authoritative is explicitly Simon's call, not
|
||||
something to assume; see `CLAUDE.md`.
|
||||
@@ -0,0 +1,175 @@
|
||||
{
|
||||
"report": "PRODUCTION_EVIDENCE.json",
|
||||
"companion_doc": "IMPLEMENTATION_REPORT.md",
|
||||
"generated_at": "2026-09-01T21:10:00+02:00",
|
||||
"repo": {
|
||||
"remote": "https://git.siax.io/sax3l/0pt.git",
|
||||
"canonical_remote_per_claude_md": "git.siax.io/sax3l/0pt",
|
||||
"note": "A second remote (github.com/sax3l/0pt) also claims to be this repo per CLAUDE.md; explicitly unresolved and left to Simon. This pass worked only against git.siax.io."
|
||||
},
|
||||
"branch": {
|
||||
"name": "feat/p010-data-os-completion",
|
||||
"renamed_from": "feat/p010-source-registry-persistence",
|
||||
"base_branch": "main",
|
||||
"base_commit": "f8bb0d5ab9f35c652d09f7639d20458cfe849f56",
|
||||
"head_commit": "9740e20818a8ce0a90498bdb12c501bc74395ac",
|
||||
"commits_on_branch": [
|
||||
{
|
||||
"sha": "07b13bd8331017462ead11b9b2ce8fa9188e520f",
|
||||
"summary": "feat(data-os): Source Registry Postgres persistence + full HTTP route surface"
|
||||
},
|
||||
{
|
||||
"sha": "9740e20818a8ce0a90498bdb12c501bc74395ac",
|
||||
"summary": "feat(data-os): TS/Python quality bridge + provenance fields + fault-injection tests"
|
||||
}
|
||||
],
|
||||
"pushed_to_origin_this_pass": true,
|
||||
"diffstat_vs_base": {
|
||||
"files_changed": 47,
|
||||
"insertions": 3119,
|
||||
"deletions": 201
|
||||
}
|
||||
},
|
||||
"pull_request": {
|
||||
"url": "https://git.siax.io/sax3l/0pt/pulls/28",
|
||||
"number": 28,
|
||||
"base": "main",
|
||||
"head": "feat/p010-data-os-completion",
|
||||
"state_at_report_time": "open",
|
||||
"opened_this_pass": true
|
||||
},
|
||||
"verification_methodology": {
|
||||
"description": "All numbers below were produced by a from-scratch verification pass: a fresh `git clone` of the pushed branch into an isolated directory, a fresh `pnpm install --frozen-lockfile`, and a fresh Python venv built from requirements-dev.txt. This is deliberately NOT a rerun inside the original working tree that developed the branch, to avoid reporting stale or environment-contaminated results.",
|
||||
"clean_clone_path_used_for_verification": "isolated scratch directory, discarded after this pass",
|
||||
"original_working_tree": "/home/simon/siax-platforms-work/0pt"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"name": "pnpm install --frozen-lockfile",
|
||||
"result": "ok",
|
||||
"note": "lockfileVersion 9, no resolution drift from the committed pnpm-lock.yaml"
|
||||
},
|
||||
{
|
||||
"name": "typecheck",
|
||||
"command": "pnpm run typecheck",
|
||||
"underlying": "tsc -p tsconfig.check.json (src/**/*.ts + tests/**/*.ts)",
|
||||
"ci_gate": "blocking",
|
||||
"result": "pass",
|
||||
"errors": 0,
|
||||
"note": "Zero errors on the pristine clone. The original dev working tree separately contains 5 untracked/uncommitted stray paths (src/fleet/, src/graph/, src/metrics/, src/projections/, src/features/feature-registry.ts) that were already present before this pass (flagged in BEFORE_STATE.md as unaudited) and are NOT part of any commit on this branch or on main -- confirmed via `git diff --stat origin/main..feat/p010-data-os-completion` touching none of them, and via their total absence from the fresh clone. Running the same command inside that non-clean working tree surfaces 14 TS errors from those 5 files; that is local litter CI will never see, not a defect in this PR."
|
||||
},
|
||||
{
|
||||
"name": "unit/integration tests (TypeScript)",
|
||||
"command": "pnpm run test",
|
||||
"underlying": "vitest run",
|
||||
"ci_gate": "blocking",
|
||||
"result": "pass",
|
||||
"test_files_passed": 34,
|
||||
"test_files_skipped": 3,
|
||||
"tests_passed": 256,
|
||||
"tests_skipped": 18,
|
||||
"tests_new_this_branch": 7,
|
||||
"new_test_file": "tests/p0/quality-bridge.test.ts",
|
||||
"skip_reason": "pre-existing .realinfra/.integration suites self-skip honestly when no live ST0RE/OpenSearch/ClickHouse is reachable from the verification environment",
|
||||
"sample_skip_stderr": "[realinfra] ST0RE (https://api.st0re.siax.io) not reachable this run (ST0RE responded HTTP 401) -- skipping honestly, not fabricating success."
|
||||
},
|
||||
{
|
||||
"name": "lint",
|
||||
"command": "pnpm run lint",
|
||||
"underlying": "eslint src tests",
|
||||
"ci_gate": "advisory (continue-on-error in both .gitea/workflows/ci.yml and .github/workflows/ci.yml)",
|
||||
"result": "pre-existing failures only, zero introduced by this branch",
|
||||
"errors": 12,
|
||||
"warnings": 4,
|
||||
"verification": [
|
||||
"git diff --stat origin/main..feat/p010-data-os-completion on every flagged file returns no output -- byte-identical to main",
|
||||
"ran `pnpm lint` against a worktree checked out directly at origin/main and got the identical 12 errors / 4 warnings"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "build",
|
||||
"command": "pnpm run build",
|
||||
"underlying": "next build",
|
||||
"ci_gate": "blocking in .gitea/workflows/ci.yml; not run by .github/workflows/ci.yml (needs runtime secrets Vercel owns)",
|
||||
"result": "pass",
|
||||
"note": "Route manifest confirms new surface is built: /api/0pt/quality/{bridge-run,issues,rules,rules/[ruleId],runs,runs/[runId]} and /api/0pt/sources/{route,[sourceId],[sourceId]/{freshness,health,provenance,rights}} all present as dynamic routes."
|
||||
},
|
||||
{
|
||||
"name": "Python engine tests",
|
||||
"command": "python -m pytest -v",
|
||||
"cwd": "services/api",
|
||||
"environment": "fresh venv, python3.12, requirements-dev.txt",
|
||||
"ci_gate": "NOT WIRED INTO CI -- both .gitea/workflows/ci.yml and .github/workflows/ci.yml state the Python engine is gated separately if/when a Python workflow is added; this pass did not add that workflow",
|
||||
"result": "pass",
|
||||
"tests_passed": 28,
|
||||
"tests_skipped": 2,
|
||||
"skip_detail": [
|
||||
{
|
||||
"test": "test_cdc_interruption_not_implementable_against_real_code",
|
||||
"reason": "no CDC pipeline exists anywhere in services/api"
|
||||
},
|
||||
{
|
||||
"test": "test_true_backfill_idempotency_not_implementable_against_real_code",
|
||||
"reason": "no delivery/order persistence layer exists anywhere in services/api"
|
||||
}
|
||||
],
|
||||
"new_test_files": [
|
||||
"services/api/tests/test_connectors_provenance.py",
|
||||
"services/api/tests/test_known_gaps.py",
|
||||
"services/api/tests/test_pipeline_fault_injection.py",
|
||||
"services/api/tests/test_quality_bridge.py",
|
||||
"services/api/tests/test_resolve_determinism.py"
|
||||
]
|
||||
}
|
||||
],
|
||||
"net_ci_status": {
|
||||
"gitea_ci_blocking_gates": {
|
||||
"typecheck": "green",
|
||||
"test": "green",
|
||||
"build": "green"
|
||||
},
|
||||
"github_ci_blocking_gates": {
|
||||
"typecheck": "green",
|
||||
"test": "green"
|
||||
},
|
||||
"advisory_only_not_blocking": ["lint (both workflows)", "typecheck:app (github only)"],
|
||||
"unwired_gate_not_in_ci": ["python pytest (services/api) -- manually verified green, no automated workflow yet"]
|
||||
},
|
||||
"production_feature_flags": {
|
||||
"OPT_QUALITY_BRIDGE_ENABLED": {
|
||||
"default_shipped_value": "unset (treated as false)",
|
||||
"changed_by_this_pr": false,
|
||||
"activated_in_production_by_this_pass": false,
|
||||
"behavior_when_off": "quality_bridge.py makes zero network calls, returns status=disabled immediately; pipeline.py's delivery decision is completely unaffected",
|
||||
"behavior_when_on_but_bridge_unreachable": "fails open -- status=unavailable, delivery decision unaffected, error never raised into run_job",
|
||||
"behavior_when_on_and_reachable": "advisory only -- stamps a quality field on delivered provenance and records a score; does NOT and, per the current ExpectationEngine contract, currently CANNOT change which rows pipeline.py's confidence-threshold filter keeps or drops",
|
||||
"decision_owner": "Simon",
|
||||
"readiness_checklist_before_enabling": [
|
||||
"OPT_QUALITY_BRIDGE_URL / OPT_QUALITY_BRIDGE_EDGE_URL must point at a reachable, authenticated 0pt.app deployment (cross-service network dependency -- Python engine and Next.js edge are typically different deploy targets)",
|
||||
"accept that this remains advisory-only (no per-row gating) unless the ExpectationEngine contract is separately extended to return per-row results",
|
||||
"recommended: enable against one non-production tenant/order first and confirm bridge-run round-trips before any production enablement"
|
||||
],
|
||||
"manual_live_verification_performed_this_pass": {
|
||||
"what": "real `next dev` server (AUTH_ENFORCE=false, local only) + real Python pipeline hitting real brreg.no + real bridge-run round-trip + real QualityService score returned",
|
||||
"when": "during development of commit 9740e20, 2026-09-01",
|
||||
"repeatable_automated_equivalent": "services/api/tests/test_quality_bridge.py (respx-mocked bridge responses, run as part of the 28/28 pytest total above)",
|
||||
"note": "the live run was a one-off manual verification during development, not a formal staging rollout step, and not itself part of the from-scratch CI-mirroring verification in this report"
|
||||
}
|
||||
}
|
||||
},
|
||||
"residual_gaps": [
|
||||
"Python engine tests (28/28) not wired into any CI workflow yet",
|
||||
"bridge_provenance as a single unified TS/Python construct still does not exist -- this pass narrows, does not close, the provenance-schema split",
|
||||
"per-row quality gating from TS into Python is not implementable against the current ExpectationEngine contract -- advisory-only is the ceiling of this pass, not a corner cut",
|
||||
"resolve.py's greedy merge remains genuinely order-sensitive for non-transitive-similarity triangles -- tested and documented, not fixed (algorithm change, out of scope)",
|
||||
"CDC-interruption and true delivery-store backfill-idempotency tests remain unimplementable -- no CDC pipeline or delivery persistence layer exists anywhere in services/api",
|
||||
"5 untracked/unaudited local paths (src/fleet/, src/graph/, src/metrics/, src/projections/, src/features/feature-registry.ts) left exactly as found -- not part of this diff, absent from a fresh clone",
|
||||
"services/scrapers/ (WAF-bypass stack) remains on main with its deletion PR still open -- unrelated to and untouched by this pass"
|
||||
],
|
||||
"process_notes": {
|
||||
"pr_open_before_this_pass": false,
|
||||
"before_state_md_left_untouched": true,
|
||||
"gitignore_additions": [".venv/", ".pytest_cache/"],
|
||||
"leftover_processes_confirmed_stopped": true
|
||||
}
|
||||
}
|
||||
@@ -58,14 +58,25 @@ cd services/api && pip install -r requirements.txt && uvicorn orchestrator.main:
|
||||
(already committed, see [`Dockerfile`](Dockerfile)) plus [`.gitea/workflows/cd.yml`](.gitea/workflows/cd.yml)
|
||||
(renamed from `deploy.yml` 2026-08-10) that POSTs a restart to a Coolify app UUID on
|
||||
`COOLIFY_TOKEN` push.
|
||||
- **Actually live right now:** the only running instance of `0pt.app` is still on **Vercel**
|
||||
- ~~**Actually live right now:** the only running instance of `0pt.app` is still on **Vercel**
|
||||
(project `0pt`, scope *Simon Axelsson's projects*) — no Coolify app has been provisioned for
|
||||
this repo yet, so `deploy.yml`'s app UUID is a placeholder that fails the job on purpose until
|
||||
someone fills it in. Flipping the config declares intent; it does not move traffic. See
|
||||
someone fills it in. Flipping the config declares intent; it does not move traffic.~~ See
|
||||
[`docs/architecture/ARCHITECTURE.md`](docs/architecture/ARCHITECTURE.md) §5/§8 for the full
|
||||
"Vercel or fleet" state and what's still open (Coolify app provisioning + DNS cutover — a
|
||||
separate decision from this config change, since the live Vercel deployment is not being
|
||||
touched).
|
||||
- **CORRECTED 2026-09-06 — the bullet above is no longer true. The Coolify deploy hook is LIVE.**
|
||||
The repo secret `COOLIFY_TOKEN` exists (created 2026-08-31T11:34:34Z) and the repo Actions
|
||||
variable `COOLIFY_APP_ID` holds a real 24-character UUID, **not** the
|
||||
`TODO-no-coolify-app-provisioned-yet` literal. Both guard conditions in
|
||||
[`.gitea/workflows/cd.yml`](.gitea/workflows/cd.yml) therefore pass. Measured on the push to
|
||||
`main` at `1ff34410` (2026-09-06): job `Deploy to Coolify / deploy` (run 11695, job 26295)
|
||||
finished **success** with `coolify deploy HTTP 200` in its log. **Every merge into `main` now
|
||||
fires a real redeploy against `cloud.siax.io` — know what you are shipping before you merge.**
|
||||
**Probed 2026-09-06 08:00 UTC:** `0pt.app` still answers from **Vercel** (`server: Vercel`, `x-vercel-id: arn1::…`) and returns **HTTP 402 `DEPLOYMENT_DISABLED` — "Payment required"** on both `/` and `/api/health`. So the public domain is *down*, DNS has **not** been cut over to Coolify, and the Coolify app — whatever state it is in — serves no `0pt.app` traffic. Two separate facts, both true at once: the deploy hook fires for real, and the product domain is dark behind an unpaid Vercel deployment.
|
||||
Still not established: whether the Coolify app itself builds and starts — the deploy endpoint
|
||||
returning 200 only means the request was accepted, not that a container came up healthy.
|
||||
- **Engine + DB + embeds** → one VPS via [`infra/docker-compose.yml`](infra/docker-compose.yml) (Postgres+pgvector, Redis, api, worker, NocoDB, Metabase, Caddy).
|
||||
- Env inventory: [`.env.example`](.env.example).
|
||||
- **Auth on `/api/0pt/*` is fail-closed**: with `AUTH_ENFORCE` unset the middleware
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
# 0pt — capability status (Master-DoD §56 status honesty)
|
||||
|
||||
> Classification scale (Master-DoD §2, exactly one per capability):
|
||||
> `MISSING → DOCUMENTED_ONLY → SKELETON → MOCK_ONLY → IMPLEMENTED → INTEGRATION_TESTED → LIVE_VERIFIED → PRODUCTION_READY`.
|
||||
> Only `PRODUCTION_READY` may be called DONE. Docs ≠ implementation.
|
||||
> Last full audit + re-classification: **2026-08-31** (SIAX Platforms P0 wave 2).
|
||||
|
||||
## How to read this
|
||||
|
||||
- **INTEGRATION_TESTED** here means: verified against a REAL dependency
|
||||
(live OpenSearch/ClickHouse container, or a recorded wire-contract double
|
||||
pinned to the sibling repo's real handler code on its `main`) in CI.
|
||||
- **LIVE_VERIFIED** requires a verification against the actual SIAX fleet
|
||||
deployment. No new capability in this wave claims it.
|
||||
- The unit/contract/integration suites are in `tests/` and run in
|
||||
`.gitea/workflows/ci.yml` (jobs `quality` + `integration`).
|
||||
|
||||
## Cross-product clients (Master-DoD §10 "Cross-product clients")
|
||||
|
||||
| Capability | Status | Evidence | Honest gaps |
|
||||
|---|---|---|---|
|
||||
| CL0UD client (policy evaluate, `POST /api/policy/evaluate`) | **INTEGRATION_TESTED** (contract) | `src/clients/cl0ud-client.ts`; `tests/contracts/cl0ud-client.contract.test.ts` pins the real camelCase `{data}` envelope; fail-closed (missing decision ⇒ deny) | Not yet exercised against a live CL0UD deployment (no reachable CL0UD URL this session ⇒ not LIVE_VERIFIED) |
|
||||
| CL0UD capability discovery (`GET /api/capabilities`) | **INTEGRATION_TESTED** (contract) | same suite | 0pt's own capabilities are not yet registered in CL0UD's registry (§3 gap) |
|
||||
| ST0RE client (artifact register, camelCase `/v1/st0re/artifacts/register`) | **INTEGRATION_TESTED** (contract) | `src/clients/st0re-client.ts`; `tests/contracts/st0re-client.contract.test.ts` | Raw-bytes upload still goes via LocalBlobStore interim (pre-existing, unchanged); presign upload path not consumed yet |
|
||||
| N0D client (create/get job, get result) | **INTEGRATION_TESTED** (contract) | `src/clients/n0d-client.ts`; `tests/contracts/n0d-client.contract.test.ts` pins `required_role`/`resource_requirements`/body `idempotency_key` | CloudEvents completion callback receiving endpoint not built in 0pt (poll path works) |
|
||||
| AUD0 client (events + OpenLineage ingest) | **INTEGRATION_TESTED** (contract) | `src/clients/aud0-client.ts`; `tests/contracts/aud0-client.contract.test.ts`; legacy lineage shape converts to real OpenLineage | No durable local outbox for AUD0 outages (§70 "durable local/outbox path" — emission failure is surfaced, not queued) |
|
||||
| **No production mock fallback (§3/§10)** | **INTEGRATION_TESTED** | `src/platform/dependency-mode.ts`; `tests/contracts/dependency-mode.contract.test.ts` drives NODE_ENV=production for all four factories ⇒ hard `DEPENDENCY_UNAVAILABLE` | — |
|
||||
|
||||
**Correction of prior state (2026-08-31):** the pre-existing clients' HTTP
|
||||
paths were verified against the sibling repos' actual `main` branches and
|
||||
THREE of four were wrong: cl0ud (`/v1/cl0ud/policies/evaluate` — never
|
||||
existed), st0re (snake_case body — would 400), aud0 lineage
|
||||
(`/v1/aud0/lineage/events` — never existed). All fixed and pinned by
|
||||
contract tests. The earlier MOCK_ONLY classification in the README was
|
||||
accurate.
|
||||
|
||||
## Data-OS capabilities (Master-DoD §10)
|
||||
|
||||
| Capability | Status | Evidence | Honest gaps |
|
||||
|---|---|---|---|
|
||||
| Source Registry — register/query/describe/health, rights/license/purpose, freshness, provenance | **IMPLEMENTED** | `src/services/source-registry-service.ts`, `src/types/source-registry.ts`; `tests/p0/source-registry.test.ts` (fail-closed purpose gate, cross-tenant NOT_FOUND) | In-memory persistence (per-process); no Postgres store yet — production boot must therefore configure one before this counts further. No API route surface yet |
|
||||
| Source rate-policy | **IMPLEMENTED** (pre-existing) | `src/services/source-rate-policy-service.ts` | unchanged this wave |
|
||||
| Search — OpenSearch adapter, index defs, bulk, tenant + purpose/rights filter, rebuild, generation switch | **INTEGRATION_TESTED** | `src/search/*`; unit `tests/p0/search-service.test.ts`; **real OpenSearch 2.11** `tests/integration/search-opensearch.integration.test.ts` incl. projection-disaster rebuild (§68-style) — runs in CI service container | Hybrid (vector) query not built; no metrics emission; index definitions in-memory |
|
||||
| Quality — versioned rules, runs, results, issues, score, trend, GE interchange | **IMPLEMENTED** (local engine) / **SKELETON** (GE delegation) | `src/quality/*`; `tests/p0/quality-service.test.ts`, `tests/p0/expectation-engine` cases | Local engine covers 7 GE core expectation types; everything else delegates via N0D job `0pt.quality.ge_run` whose EXECUTOR (0pt-motor GE runner) does not exist yet — delegated runs stay "delegated" until it does. Never fabricates results |
|
||||
| Dataset — create (pre-existing) + version/query/export with `st0re:artifact:` refs | **IMPLEMENTED** | `src/services/dataset-version-service.ts`; `tests/p0/dataset-version-service.test.ts` (idempotency §53, canonical refs §5) | Export assumes bytes already written by a job; no route surface; in-memory persistence |
|
||||
| Lineage — OpenLineage emission to AUD0's real ingest | **INTEGRATION_TESTED** (contract) | `src/lineage/*`; aud0 contract suite; emitted from dataset version/export, quality runs, search rebuilds | Emission is best-effort (never-throw); no outbox. AUD0-side ingestion verified against contract double, not live AUD0 |
|
||||
| Metadata — OpenMetadata projection (never canonical) | **IMPLEMENTED** | `src/metadata/openmetadata-adapter.ts`; `tests/contracts/metadata-features.contract.test.ts`; `projectAll` rebuild path | No live OpenMetadata exercised (no local container this pass); lineage/glossary/ownership projection not built — only dataset entities |
|
||||
| Analytics — ClickHouse, tenant-isolated, query limits, retention | **INTEGRATION_TESTED** | `src/analytics/clickhouse-analytics.ts`; **real ClickHouse 24.8** `tests/integration/analytics-clickhouse.integration.test.ts` (tenant isolation, closed DSL, TTL schema) — runs in CI service container | Closed metric-DSL only (no ad-hoc queries by design); no quota integration with CL0UD |
|
||||
| Features — registry + Feast adapter | **IMPLEMENTED** (online path, registry) / **SKELETON** (materialize/get-historical) | `src/features/feast-adapter.ts`; contract suite pins Feast's `get-online-features` wire shape | No live Feast server exercised; offline/materialization are N0D delegation contracts whose executor (0pt-motor) does not exist yet; point-in-time correctness therefore NOT yet proven |
|
||||
| Entity Resolution — candidate API, blocking, explanation, hard-rule hook, review handoff | **IMPLEMENTED** | `src/er/entity-resolution-service.ts`; `tests/p0/entity-resolution.test.ts` (hard rules override both directions, review band, blocking refusals) | Generic reference scorer (Jaccard+Levenshtein), NOT Splink probabilistic linkage — that is the `0pt.er.batch_resolution` N0D contract, executor pending in 0pt-motor. No persistence of review queues |
|
||||
| Graph (§10 Graph) | **MISSING** | — | Not started this wave (explicitly out of the delivered P0 subset; next in line per §10) |
|
||||
|
||||
## Canonical platform plumbing
|
||||
|
||||
| Capability | Status | Evidence |
|
||||
|---|---|---|
|
||||
| SiaxRequestContext + propagation headers (mirror of cl0ud `@siax/contract`) | **INTEGRATION_TESTED** (contract-pinned) | `src/platform/siax-context.ts`; `tests/contracts/siax-context.contract.test.ts` |
|
||||
| Canonical 12-code error model + typed `SiaxError` (§6) | **INTEGRATION_TESTED** (contract-pinned) | `src/platform/errors.ts`; same suite |
|
||||
| Canonical resource refs (§5) | **IMPLEMENTED** | `src/platform/resource-ref.ts`; `tests/p0/dataset-version-service.test.ts` |
|
||||
|
||||
## Pre-existing capabilities (unchanged this wave — prior classifications stand)
|
||||
|
||||
| Capability | Status | Note |
|
||||
|---|---|---|
|
||||
| Iceberg write boundary via ST0RE lakehouse | LIVE_VERIFIED (2026-07-13) | `docs/ICEBERG_WRITE_BOUNDARY.md`; untouched |
|
||||
| Document capture/extraction | IMPLEMENTED | LocalBlobStore interim persists bytes; ST0RE raw upload still pending |
|
||||
| Edge order/payment app (Next.js) | see `docs/architecture/ARCHITECTURE.md` | outside this wave's scope |
|
||||
| FastAPI engine (`services/api`) | IMPLEMENTED | outside this wave's scope |
|
||||
| `services/scrapers/*` (WAF-bypass stack) | **quarantined, pending deletion PR** | `feature/0pt-remove-noncompliant-scrapers` — NOT touched by this wave; nothing new wires it |
|
||||
|
||||
## Global gaps blocking PRODUCTION_READY (§3, tracked, not hidden)
|
||||
|
||||
1. Persistent stores: the new registries (sources/rights/quality
|
||||
rules/dataset versions/index definitions/feature views) are per-process
|
||||
in-memory. `resolveStoreMode` guarantees production boot FAILS rather
|
||||
than serving Maps, but the Postgres stores themselves are not built.
|
||||
2. Capability registration in CL0UD + API0 discovery for the new
|
||||
capabilities (§3 "Capability är registrerad i CL0UD Capability Registry").
|
||||
3. AUD0 outbox for evidence durability under AUD0 outage (§70).
|
||||
4. Surface exposure: no `app/api/0pt/*` routes yet for
|
||||
search/quality/datasets-versions/ER (services + contracts only).
|
||||
5. 0pt-motor executors for the delegated jobs
|
||||
(`0pt.quality.ge_run`, `0pt.features.*`, `0pt.er.batch_resolution`).
|
||||
6. Observability: no metrics/traces emitted by the new services (§36).
|
||||
|
||||
---
|
||||
|
||||
## P060-P068 ingestion plane (added 2026-09-04)
|
||||
|
||||
Classification scale unchanged. Every row below states what was actually
|
||||
verified, and by which command.
|
||||
|
||||
| Requirement | BEFORE this pass | Status now | Evidence |
|
||||
|---|---|---|---|
|
||||
| **P060** Source Registry: durable canonical registry; restart preserves sources; an unclassified source cannot be activated | **PARTIAL / BROKEN.** The registry existed with Postgres persistence (`sql/002_source_registry.sql`, `src/repositories/postgres-source-registry-repository.ts`) and carried source_id/tenant/project/type/status/license/purpose/rights. It had NO authorization, NO schedule, NO checkpoint, NO schema, and no `owner` distinct from `created_by`. The activation DoD was inverted: `postgres-source-repository.ts:70` wrote `${input.status ?? "active"}`, so a source with no rights, no provenance and no authorization was ACTIVE the instant it was registered, and no activation operation existed at all | **INTEGRATION_TESTED** | `sql/007_ingestion_plane.sql` (authorization/schedule/schemas/checkpoints + `ALTER … SET DEFAULT 'pending'` + `owner_id`); `SourceRegistryService.classifySource`/`activateSource`; `tests/p0/source-activation-gate.test.ts` (11); `tests/api/0pt-p060-routes.test.ts` (9); `tests/p0/ingestion-plane-postgres.realinfra.test.ts` against a live Postgres 16 |
|
||||
| **P061** Connector contract: discover/validate/connect/checkpoint/read/resume/backfill/health/disconnect; two heterogeneous connectors pass one conformance suite | **MISSING.** `services/api/sources/connectors.py` had per-source `fetch_*()` functions only — no lifecycle, no cursor, no resume, no health, no disconnect, no conformance suite | **INTEGRATION_TESTED** | `src/connectors/connector.ts` (all nine operations); `src/connectors/conformance.ts` (15 checks); `HttpJsonConnector` (real node:http server, opaque cursor) and `JsonlFileConnector` (real file, line-offset cursor, CDC deletes) both pass the identical suite — `tests/p0/connector-conformance.test.ts`, which also proves the suite FAILS a fail-open connector |
|
||||
| **P062** CDC/batch/stream normalized to one envelope; restart resumes from a durable checkpoint without silent gaps | **MISSING**, and documented as such: `services/api/tests/test_known_gaps.py:10-19` skipped the CDC case because no CDC pipeline existed anywhere, and `workers/pipeline.py` had no persistence at all | **INTEGRATION_TESTED** | `src/types/ingestion.ts` (one envelope for all three modes); `src/ingestion/ingestion-runner.ts`; `tests/p0/ingestion-runner.test.ts` kills a run inside the sink and proves the resumed run covers all 10 records exactly once; same proof against real Postgres across fresh processes |
|
||||
| **P063** Schema versioning + compatibility policy; a breaking schema is detected before consumer corruption | **MISSING.** `schema_version` existed as a free-form string on datasets/Iceberg writes; there was no comparison, no policy, no versioned source schema | **INTEGRATION_TESTED** | `src/schema/compatibility.ts`, `src/schema/schema-registry.ts`; `tests/p0/schema-evolution.test.ts` (12); the runner's schema gate runs before the first read — `tests/p0/ingestion-runner.test.ts` asserts `sinkCalls === 0` on a break |
|
||||
| **P064** Quality rules behind the SIAX abstraction; failures block or quarantine per declared policy | **PARTIAL.** `src/quality/*` had rules, runs, results, issues, scores and a GE-compatible engine, but a failing rule only opened an ISSUE — the data flowed on regardless. `grep -ri quarantin src/` found one unrelated hit | **INTEGRATION_TESTED** | `src/quality/quality-policy.ts` composes the EXISTING `ExpectationEngine` (not a second engine) and adds the consequence; block / quarantine / warn each proved in `tests/p0/ingestion-runner.test.ts`; quarantine durability proved against real Postgres |
|
||||
| **P065** Normalization/ER: only the canonical highway; no second simplified production engine | **DUPLICATE (confirmed).** `src/er/entity-resolution-service.ts` (canonical: hard rules, review band, Splink delegation) AND `services/api/workers/resolve.py` (simplified fuzzy threshold, auto-merge) — the latter still called by `workers/pipeline.py:110` | **NOT DONE — documented, not built.** The duplication is now stated at BOTH sites in code with the removal plan. The new ingestion plane deliberately performs no ER, so it adds no third engine. See "P065 — why it was not closed" below |
|
||||
| **P066** Iceberg | out of scope — ST0RE is owned by another workflow | **NOT TOUCHED** | Contract read only (`src/services/iceberg-write-service.ts`, `src/clients/st0re-lakehouse-client.ts`) |
|
||||
| **P067** Lineage: OpenLineage/AUD0 events; source → output lineage is queryable | **PARTIAL.** `src/lineage/lineage-emitter.ts` emitted OpenLineage to AUD0, but nothing in this repo could ANSWER a lineage question — no store, no query, no route | **INTEGRATION_TESTED** | `src/lineage/lineage-edge.ts`, `src/lineage/lineage-store.ts` (local queryable projection; AUD0 stays the system of record), `GET /api/0pt/lineage`; `tests/p0/ingestion-runner.test.ts` queries both directions and proves tenant scoping |
|
||||
| **P068** Durable bounded backfill/replay; repeated backfill is idempotent | **MISSING**, documented as such: `services/api/tests/test_known_gaps.py:44-52` skipped it because "no delivery-store persistence layer exists to dedupe a backfill against" | **INTEGRATION_TESTED** | `opt_ingestion_ledger` + `IngestionRunner.backfill`; `tests/p0/ingestion-runner.test.ts` (repeat emits 0, live checkpoint untouched, a real content change still re-emits); same against real Postgres |
|
||||
|
||||
### What was actually run (not "should pass")
|
||||
|
||||
- `pnpm run typecheck` — clean.
|
||||
- `pnpm run test` — **365 passed, 25 skipped** (was 311/18 before this pass).
|
||||
- `pnpm run build` — clean; the four new routes appear in the route table.
|
||||
- `pnpm run lint` — 12 errors / 5 warnings, **all pre-existing** (identical
|
||||
count on the base commit; advisory in CI). This pass adds none.
|
||||
- Real Postgres 16: `tests/p0/ingestion-plane-postgres.realinfra.test.ts`
|
||||
with `INGESTION_TEST_DATABASE_URL` — 7 passed.
|
||||
- **Real container restart**, run manually against that same database:
|
||||
`docker restart` the Postgres container, then re-read through the real
|
||||
repositories. Before and after were identical — 2 sources (one `active`
|
||||
with its owner, one `pending`), 1 checkpoint (cursor 6, 6 committed),
|
||||
6 ledger rows, 1 lineage edge, 4 quarantined records, 1 schema version.
|
||||
That is P060's "a restart preserves all sources", measured rather than
|
||||
asserted.
|
||||
|
||||
### A real bug this pass wrote and then caught
|
||||
|
||||
The first draft of `IngestionRepository` combined the idempotency check and
|
||||
its recording into one atomic call, made BEFORE the sink. The crash/resume
|
||||
test then delivered **7 of 10 records**: the page in flight when the sink
|
||||
died had already been marked delivered, so the resumed run skipped it as a
|
||||
duplicate — permanent, silent data loss. The fix is the `peekDeliveries` /
|
||||
`commitDeliveries` split, with both the ledger commit and the checkpoint
|
||||
commit after the sink resolves. Recorded here because the failure mode is
|
||||
the interesting artifact, not the fix.
|
||||
|
||||
### Honest gaps in the new plane (not hidden)
|
||||
|
||||
1. **Delivery is AT-LEAST-ONCE, not exactly-once.** Two concurrent runs on
|
||||
the same (source, partition) can both peek "new" and both emit.
|
||||
`envelope_id` is deterministic so a dedup-capable sink survives it. There
|
||||
is no per-(source, partition) run lock. Do not describe this plane as
|
||||
exactly-once.
|
||||
2. **No CDC *source* exists.** The envelope, the operations and the runner
|
||||
handle change events, and `JsonlFileConnector` emits real deletes from an
|
||||
`_op` field — but this repo still has no Debezium/WAL/replication-slot
|
||||
consumer. P062's contract is met; a CDC connector is still MISSING.
|
||||
3. **0pt does not schedule itself.** `opt_source_schedule` is a declaration
|
||||
ACT0 would read; there is no cron daemon here.
|
||||
4. **The sink is caller-supplied.** The runner stops at the sink; the
|
||||
lakehouse write boundary is ST0RE's (P066, another workflow's repo).
|
||||
5. **Nothing here is deployed.** See the deployment note below.
|
||||
6. **`QualityService`'s own rule/run/issue registry is still in-process
|
||||
Maps** (`src/quality/quality-service.ts:47-50`). The new *enforcement*
|
||||
path is durable; the older rule registry is not. Unchanged by this pass.
|
||||
7. **Python tests still run nowhere.** `grep -rn "pytest" .gitea/workflows
|
||||
.github/workflows` finds nothing — `services/api/tests/*` (9 files) has
|
||||
never been executed by CI. Flagged, not fixed (it needs a workflow change
|
||||
and a Python toolchain decision).
|
||||
|
||||
### Deployment reality, measured 2026-09-04
|
||||
|
||||
`https://0pt.app/api/health` answers **200** from **Vercel** with
|
||||
`{"db":"not_configured"}`, and `https://0pt.app/api/0pt/sources` answers
|
||||
**404** — the deployed build predates the entire `app/api/0pt/*` capability
|
||||
layer. `https://api.0pt.app/health` answers **404** (the FastAPI engine is
|
||||
not reachable), and `.gitea/workflows/cd.yml` still guards on
|
||||
`COOLIFY_APP_ID = TODO-no-coolify-app-provisioned-yet`. So: none of the
|
||||
capability layer — neither this pass's work nor P010/P014's — is running in
|
||||
production, and P060's durability guarantee cannot be exercised there,
|
||||
because production has no database configured and does not serve the routes.
|
||||
Verified with `curl`; not inferred.
|
||||
|
||||
### P065 — why it was not closed
|
||||
|
||||
Closing P065 means turning `services/api/workers/resolve.py` into an adapter
|
||||
over the canonical `src/er/entity-resolution-service.ts`. That change cannot
|
||||
be verified from this repo today: `services/api` is not built by this repo's
|
||||
Dockerfile (it builds only the Next.js app), `api.0pt.app` answers 404, and
|
||||
no CI workflow runs its pytest suite. A blind rewrite would ship untested
|
||||
into a path nobody can exercise — the exact failure mode this programme
|
||||
exists to stop.
|
||||
|
||||
The removal plan, for whoever picks it up:
|
||||
1. Expose the canonical ER capability as `POST /api/0pt/er/candidates`
|
||||
(service already exists; only a route + Zod schema are missing).
|
||||
2. Add a Python CI job that actually runs `services/api/tests`.
|
||||
3. Replace `resolve.py`'s body with an HTTP adapter over that route,
|
||||
**fail-closed** when it is unreachable — never falling back to the local
|
||||
scorer, which would preserve the duplicate under a different name.
|
||||
4. Port `test_resolve_determinism.py` to assert the adapter's contract, and
|
||||
delete the fuzzy scorer.
|
||||
@@ -1,6 +1,7 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, documentExtractionSchema } from "@/src/validation/0pt-schemas";
|
||||
import { documentExtractionService } from "@/src/services/document-extraction-service";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
export async function POST(request: NextRequest, { params }: { params: Promise<{ documentId: string }> }) {
|
||||
try {
|
||||
@@ -11,6 +12,8 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
|
||||
return NextResponse.json(__parsed.body, { status: __parsed.status });
|
||||
}
|
||||
const body = __parsed.data;
|
||||
const tenantGuard = assertTenantMatchesVerified(request, body.tenant_id);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const result = await documentExtractionService.extractDocument(documentId, {
|
||||
tenant_id: body.tenant_id,
|
||||
project_id: body.project_id,
|
||||
@@ -35,10 +38,17 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
|
||||
}
|
||||
}
|
||||
|
||||
export async function GET(_request: NextRequest, { params }: { params: Promise<{ documentId: string }> }) {
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ documentId: string }> }) {
|
||||
try {
|
||||
const { documentId } = await params;
|
||||
const extractions = await documentExtractionService.listExtractionsForDocument(documentId);
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const extractions = await documentExtractionService.listExtractionsForDocument(documentId, tenantId);
|
||||
return NextResponse.json(extractions);
|
||||
} catch (error: unknown) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, documentCaptureSchema } from "@/src/validation/0pt-schemas";
|
||||
import { documentCaptureService } from "@/src/services/document-capture-service";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
@@ -10,6 +11,8 @@ export async function POST(request: NextRequest) {
|
||||
return NextResponse.json(__parsed.body, { status: __parsed.status });
|
||||
}
|
||||
const body = __parsed.data;
|
||||
const tenantGuard = assertTenantMatchesVerified(request, body.tenant_id);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const result = await documentCaptureService.captureFromMailbox(
|
||||
{
|
||||
mailbox: {
|
||||
@@ -26,8 +29,18 @@ export async function POST(request: NextRequest) {
|
||||
{
|
||||
tenant_id: body.tenant_id,
|
||||
project_id: body.project_id,
|
||||
actor_id: body.actor_id,
|
||||
correlation_id: body.correlation_id,
|
||||
// documentCaptureSchema types these optional (pre-existing, listed
|
||||
// among package.json's "9 pre-existing errors ... Wave 2 Zod
|
||||
// schemas type fields as plain string while the services want
|
||||
// narrower unions" -- deliberately left to its own change, not
|
||||
// widened here). CaptureMailboxContext requires actor_id as a
|
||||
// non-empty string and captureFromMailbox already throws a clear
|
||||
// "missing tenant_id/project_id/actor_id" error when it's falsy, so
|
||||
// `?? ""` changes no runtime behavior -- it only satisfies the type
|
||||
// checker, which previously never saw this file (nothing under
|
||||
// tests/ imported it before P014's cross-tenant regression tests).
|
||||
actor_id: body.actor_id ?? "",
|
||||
correlation_id: body.correlation_id ?? "",
|
||||
},
|
||||
);
|
||||
return NextResponse.json(result, { status: 201 });
|
||||
@@ -37,9 +50,20 @@ export async function POST(request: NextRequest) {
|
||||
}
|
||||
}
|
||||
|
||||
// P014: `tenant_id` used to be optional here, and
|
||||
// DocumentCaptureService.listDocuments treats an absent tenant_id filter as
|
||||
// "no restriction" — so calling this with no query params at all returned
|
||||
// EVERY tenant's captured documents to any authenticated caller. Every
|
||||
// sibling list/get route in this file already requires tenant_id; this was
|
||||
// the one outlier. Now required, same as the rest of the surface.
|
||||
export async function GET(request: NextRequest) {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id") ?? undefined;
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const projectId = searchParams.get("project_id") ?? undefined;
|
||||
const documents = await documentCaptureService.listDocuments({ tenant_id: tenantId, project_id: projectId });
|
||||
return NextResponse.json(documents);
|
||||
|
||||
@@ -1,10 +1,25 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { mappingService } from "@/src/services/lakehouse-mapping-registry";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
export async function GET(_request: Request, { params }: { params: Promise<{ mappingId: string }> }) {
|
||||
// P014: this lookup used to accept only `mappingId`, with NO tenant check
|
||||
// anywhere in the request path -- any authenticated caller could read
|
||||
// another tenant's dataset<->Iceberg-table mapping (namespace, table name,
|
||||
// storage URI) by guessing or enumerating IDs. Now requires `?tenant_id=`
|
||||
// and both checks it against the verified token and threads it into the
|
||||
// service so a foreign mapping_id 404s instead of leaking -- see
|
||||
// DatasetIcebergMappingService.getMapping's doc comment.
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ mappingId: string }> }) {
|
||||
try {
|
||||
const { mappingId } = await params;
|
||||
const mapping = await mappingService.getMapping(mappingId);
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const mapping = await mappingService.getMapping(mappingId, tenantId);
|
||||
return NextResponse.json(mapping);
|
||||
} catch (error: unknown) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, requestIcebergWriteSchema } from "@/src/validation/0pt-schemas";
|
||||
import { writeService } from "@/src/services/lakehouse-mapping-registry";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
// Registers a new Iceberg snapshot (a write) through ST0RE's table registry
|
||||
// for an already-mapped dataset. See docs/ICEBERG_WRITE_BOUNDARY.md.
|
||||
@@ -22,8 +23,11 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
|
||||
return NextResponse.json(__parsed.body, { status: __parsed.status });
|
||||
}
|
||||
const body = __parsed.data;
|
||||
const tenantGuard = assertTenantMatchesVerified(request, body.tenant_id);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const result = await writeService.requestIcebergWrite({
|
||||
dataset_iceberg_mapping_id: mappingId,
|
||||
tenant_id: body.tenant_id,
|
||||
actor_id: body.actor_id,
|
||||
correlation_id: body.correlation_id,
|
||||
snapshot_type: body.snapshot_type,
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, mapDatasetToIcebergTableSchema } from "@/src/validation/0pt-schemas";
|
||||
import { mappingService, writeService } from "@/src/services/lakehouse-mapping-registry";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
import type { IcebergTableFormat } from "@/src/clients/st0re-lakehouse-client";
|
||||
|
||||
// Shared instances come from src/services/lakehouse-mapping-registry.ts so a
|
||||
// mapping created via POST is visible to a later GET, AND to the sibling
|
||||
@@ -28,6 +30,8 @@ export async function POST(request: NextRequest) {
|
||||
return NextResponse.json(__parsed.body, { status: __parsed.status });
|
||||
}
|
||||
const body = __parsed.data;
|
||||
const tenantGuard = assertTenantMatchesVerified(request, body.tenant_id);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const result = await writeService.mapDatasetToIcebergTable({
|
||||
tenant_id: body.tenant_id,
|
||||
project_id: body.project_id,
|
||||
@@ -37,7 +41,17 @@ export async function POST(request: NextRequest) {
|
||||
table_name: body.table_name,
|
||||
storage_uri: body.storage_uri,
|
||||
catalog_ref: body.catalog_ref,
|
||||
table_format: body.table_format,
|
||||
// mapDatasetToIcebergTableSchema types this as a plain string
|
||||
// (pre-existing, listed among package.json's "9 pre-existing errors
|
||||
// ... Wave 2 Zod schemas type fields as plain string while the
|
||||
// services want narrower unions" -- deliberately left to its own
|
||||
// change, not widened here). This cast changes no runtime behavior:
|
||||
// an unrecognized value flows through exactly as it did before
|
||||
// (ST0RE's own table-registration route is the actual validator) --
|
||||
// it only satisfies the type checker, which previously never saw this
|
||||
// file (nothing under tests/ imported it before P014's cross-tenant
|
||||
// regression tests).
|
||||
table_format: body.table_format as IcebergTableFormat | undefined,
|
||||
actor_id: body.actor_id,
|
||||
correlation_id: body.correlation_id,
|
||||
});
|
||||
@@ -48,10 +62,21 @@ export async function POST(request: NextRequest) {
|
||||
}
|
||||
}
|
||||
|
||||
// P014: tenant_id used to be optional here, and
|
||||
// DatasetIcebergMappingService.listMappings treats an absent tenant_id
|
||||
// filter as "no restriction" -- omitting the query param returned every
|
||||
// tenant's dataset<->Iceberg-table mappings (namespace, table name, storage
|
||||
// URI). Now required and cross-checked, matching the rest of the surface.
|
||||
export async function GET(request: NextRequest) {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const mappings = await mappingService.listMappings({
|
||||
tenant_id: searchParams.get("tenant_id") ?? undefined,
|
||||
tenant_id: tenantId,
|
||||
project_id: searchParams.get("project_id") ?? undefined,
|
||||
dataset_id: searchParams.get("dataset_id") ?? undefined,
|
||||
});
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* app/api/0pt/lineage/route.ts
|
||||
* =============================
|
||||
* P067 DoD: "source -> output lineage is QUERYABLE". This is the query.
|
||||
*
|
||||
* GET /api/0pt/lineage?tenant_id=…&output_name=… -> what fed this output
|
||||
* GET /api/0pt/lineage?tenant_id=…&input_name=… -> what this input produced
|
||||
*
|
||||
* The answer comes from 0pt's local projection of the OpenLineage events it
|
||||
* emits to AUD0 (src/lineage/lineage-store.ts). AUD0 remains the system of
|
||||
* record; this endpoint exists because "emitted to AUD0" was never the same
|
||||
* thing as "answerable from 0pt", and before this route nothing in this repo
|
||||
* could answer either question.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { lineageStore } from "@/src/services/source-registry";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
const outputName = searchParams.get("output_name") ?? undefined;
|
||||
const inputName = searchParams.get("input_name") ?? undefined;
|
||||
if (!tenantId) return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
if (!outputName && !inputName) {
|
||||
return NextResponse.json(
|
||||
{ error: "one of output_name or input_name is required — an unfiltered lineage dump is not a query" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
const guard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (guard) return guard;
|
||||
const limitRaw = searchParams.get("limit");
|
||||
const limit = limitRaw ? Math.min(Math.max(Number(limitRaw), 1), 500) : undefined;
|
||||
const edges = await lineageStore.query({
|
||||
tenant_id: tenantId,
|
||||
output_name: outputName,
|
||||
input_name: inputName,
|
||||
limit,
|
||||
});
|
||||
return NextResponse.json({ edges, count: edges.length });
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* app/api/0pt/quality/bridge-run/route.ts
|
||||
* ==========================================
|
||||
* The P010 TS/Python split-brain bridge (BEFORE_STATE.md §2). This is the
|
||||
* ONE route the production Python engine
|
||||
* (services/api/workers/quality_bridge.py, called from
|
||||
* services/api/workers/pipeline.py) talks to. It:
|
||||
*
|
||||
* 1. auto-provisions a deterministic, tenant-scoped baseline rule set for
|
||||
* (tenant_id, dataset_id) the first time it's seen (src/quality/quality-bridge.ts) --
|
||||
* the Python engine never has to pre-register rules through the CRUD
|
||||
* surface just to get a quality read on an order's already-produced rows;
|
||||
* 2. runs QualityService.runQuality synchronously against the rows the
|
||||
* pipeline already produced (delivered, post the Python filter --
|
||||
* this is an ADVISORY second opinion, not a gate, until Simon decides
|
||||
* to wire per-row gating -- see quality_bridge.py's docstring for why
|
||||
* that isn't done in this pass);
|
||||
* 3. returns the run + any open issues.
|
||||
*
|
||||
* This does not replace, call, or block on the Python filter in any way --
|
||||
* it is purely additive. If this route is unreachable, times out, or 500s,
|
||||
* the Python side is required to fail open (never raise into the pipeline).
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, qualityBridgeRunSchema } from "@/src/validation/0pt-schemas";
|
||||
import { qualityService } from "@/src/services/quality";
|
||||
import { ensureBaselineRules } from "@/src/quality/quality-bridge";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const __raw = await request.json();
|
||||
const __parsed = parseBody(qualityBridgeRunSchema, __raw);
|
||||
if (!__parsed.ok) {
|
||||
return NextResponse.json(__parsed.body, { status: __parsed.status });
|
||||
}
|
||||
const body = __parsed.data;
|
||||
|
||||
await ensureBaselineRules(qualityService, {
|
||||
tenant_id: body.tenant_id,
|
||||
project_id: body.project_id,
|
||||
dataset_id: body.dataset_id,
|
||||
});
|
||||
|
||||
const run = await qualityService.runQuality({
|
||||
tenant_id: body.tenant_id,
|
||||
project_id: body.project_id,
|
||||
dataset_id: body.dataset_id,
|
||||
dataset_version: body.dataset_version,
|
||||
rows: body.rows,
|
||||
correlation_id: body.correlation_id,
|
||||
});
|
||||
|
||||
const issues = qualityService.listIssues({
|
||||
tenant_id: body.tenant_id,
|
||||
dataset_id: body.dataset_id,
|
||||
status: "open",
|
||||
});
|
||||
|
||||
return NextResponse.json({ run, issues }, { status: 201 });
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* app/api/0pt/quality/issues/route.ts
|
||||
* ======================================
|
||||
* quality.list_issues (GET) -- issues opened by finished quality runs
|
||||
* (human-review handoff, Master-DoD §10).
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { qualityService } from "@/src/services/quality";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
import type { DataQualityIssue } from "@/src/quality/quality-types";
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const issues = qualityService.listIssues({
|
||||
tenant_id: tenantId,
|
||||
dataset_id: searchParams.get("dataset_id") ?? undefined,
|
||||
status: (searchParams.get("status") as DataQualityIssue["status"]) ?? undefined,
|
||||
});
|
||||
return NextResponse.json(issues);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* app/api/0pt/quality/rules/[ruleId]/route.ts
|
||||
* ===============================================
|
||||
* quality.get_rule (GET) + quality.update_rule (PUT, versioned -- bumps
|
||||
* `version`, records the superseded row). Cross-tenant reads are NOT_FOUND
|
||||
* (no existence oracle), matching QualityService.getRule's own contract.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { qualityService } from "@/src/services/quality";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
import { z } from "zod";
|
||||
|
||||
const updateQualityRuleSchema = z.object({
|
||||
tenant_id: z.string().min(1),
|
||||
name: z.string().min(1).optional(),
|
||||
expectation: z.object({ expectation_type: z.string().min(1), kwargs: z.record(z.string(), z.unknown()) }).optional(),
|
||||
severity: z.enum(["info", "warning", "error", "critical"]).optional(),
|
||||
status: z.enum(["active", "disabled", "superseded"]).optional(),
|
||||
});
|
||||
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ ruleId: string }> }) {
|
||||
try {
|
||||
const { ruleId } = await params;
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const versions = searchParams.get("versions") === "true";
|
||||
if (versions) {
|
||||
return NextResponse.json(qualityService.getRuleVersions(ruleId, tenantId));
|
||||
}
|
||||
return NextResponse.json(qualityService.getRule(ruleId, tenantId));
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function PUT(request: NextRequest, { params }: { params: Promise<{ ruleId: string }> }) {
|
||||
try {
|
||||
const { ruleId } = await params;
|
||||
const raw = await request.json();
|
||||
const parsed = updateQualityRuleSchema.safeParse(raw);
|
||||
if (!parsed.success) {
|
||||
return NextResponse.json(
|
||||
{ error: "validation_error", issues: parsed.error.issues.map((i) => ({ path: i.path.join("."), message: i.message })) },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
const { tenant_id, ...patch } = parsed.data;
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenant_id);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const rule = await qualityService.updateRule(ruleId, tenant_id, patch);
|
||||
return NextResponse.json(rule);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* app/api/0pt/quality/rules/route.ts
|
||||
* ====================================
|
||||
* quality.create_rule (POST) + quality.list_rules (GET) -- the CRUD surface
|
||||
* over QualityService's versioned rule registry (src/quality/quality-service.ts,
|
||||
* Master-DoD §10). Uses the shared singleton from src/services/quality.ts,
|
||||
* not a fresh `new QualityService()` per route module -- see that file's
|
||||
* doc comment (and BEFORE_STATE.md §1, which found and fixed the same
|
||||
* per-module-instance defect for the Source Registry).
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, createQualityRuleSchema } from "@/src/validation/0pt-schemas";
|
||||
import { qualityService } from "@/src/services/quality";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
import type { DataQualityRule } from "@/src/quality/quality-types";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const __raw = await request.json();
|
||||
const __parsed = parseBody(createQualityRuleSchema, __raw);
|
||||
if (!__parsed.ok) {
|
||||
return NextResponse.json(__parsed.body, { status: __parsed.status });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, __parsed.data.tenant_id);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const rule = await qualityService.createRule(__parsed.data);
|
||||
return NextResponse.json(rule, { status: 201 });
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const rules = qualityService.listRules({
|
||||
tenant_id: tenantId,
|
||||
dataset_id: searchParams.get("dataset_id") ?? undefined,
|
||||
status: (searchParams.get("status") as DataQualityRule["status"]) ?? undefined,
|
||||
});
|
||||
return NextResponse.json(rules);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* app/api/0pt/quality/runs/[runId]/route.ts
|
||||
* =============================================
|
||||
* quality.get_run (GET). Cross-tenant reads are NOT_FOUND.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { qualityService } from "@/src/services/quality";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ runId: string }> }) {
|
||||
try {
|
||||
const { runId } = await params;
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
return NextResponse.json(qualityService.getRun(runId, tenantId));
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
/**
|
||||
* app/api/0pt/quality/runs/route.ts
|
||||
* ====================================
|
||||
* quality.run (POST) + quality.list_runs (GET). POST executes all active
|
||||
* rules for a dataset against caller-supplied rows -- runs synchronously
|
||||
* when every active rule's expectation is locally executable, otherwise
|
||||
* delegates to the GE backend via N0D (QualityService.runQuality never
|
||||
* fabricates a result for a check it couldn't run).
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, createQualityRunSchema } from "@/src/validation/0pt-schemas";
|
||||
import { qualityService } from "@/src/services/quality";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const __raw = await request.json();
|
||||
const __parsed = parseBody(createQualityRunSchema, __raw);
|
||||
if (!__parsed.ok) {
|
||||
return NextResponse.json(__parsed.body, { status: __parsed.status });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, __parsed.data.tenant_id);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const run = await qualityService.runQuality(__parsed.data);
|
||||
return NextResponse.json(run, { status: 201 });
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const runs = qualityService.listRuns({
|
||||
tenant_id: tenantId,
|
||||
dataset_id: searchParams.get("dataset_id") ?? undefined,
|
||||
});
|
||||
return NextResponse.json(runs);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* app/api/0pt/sources/[sourceId]/activate/route.ts
|
||||
* ==================================================
|
||||
* P060 activation gate over HTTP. POST activates a source ONLY when it is
|
||||
* classified (owner + unexpired authorization + rights + provenance);
|
||||
* otherwise it returns 403 AUTHORIZATION_DENIED naming every missing fact.
|
||||
* GET returns the classification without changing anything, so an operator
|
||||
* can see what is missing before trying.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { activateSourceSchema, parseBody } from "@/src/validation/0pt-schemas";
|
||||
import { sourceRegistryService } from "@/src/services/source-registry";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const tenantId = new URL(request.url).searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const guard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (guard) return guard;
|
||||
return NextResponse.json(await sourceRegistryService.classifySource(sourceId, tenantId));
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const parsed = parseBody(activateSourceSchema, await request.json());
|
||||
if (!parsed.ok) return NextResponse.json(parsed.body, { status: parsed.status });
|
||||
const guard = assertTenantMatchesVerified(request, parsed.data.tenant_id);
|
||||
if (guard) return guard;
|
||||
const source = await sourceRegistryService.activateSource(
|
||||
sourceId,
|
||||
parsed.data.tenant_id,
|
||||
parsed.data.actor_id,
|
||||
);
|
||||
return NextResponse.json(source);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* app/api/0pt/sources/[sourceId]/authorization/route.ts
|
||||
* ======================================================
|
||||
* P060 "authorization": the recorded decision that 0pt may connect to this
|
||||
* source at all — separate from rights (what the DATA may be used for). PUT
|
||||
* is an upsert; a service identity as `granted_by` is rejected (400), because
|
||||
* a source may not authorize itself.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, setSourceAuthorizationSchema } from "@/src/validation/0pt-schemas";
|
||||
import { sourceRegistryService } from "@/src/services/source-registry";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const tenantId = new URL(request.url).searchParams.get("tenant_id");
|
||||
if (!tenantId) return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
const guard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (guard) return guard;
|
||||
const auth = await sourceRegistryService.getAuthorization(sourceId, tenantId);
|
||||
if (!auth) {
|
||||
return NextResponse.json({ error: `source ${sourceId} has no authorization record` }, { status: 404 });
|
||||
}
|
||||
return NextResponse.json(auth);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function PUT(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const parsed = parseBody(setSourceAuthorizationSchema, await request.json());
|
||||
if (!parsed.ok) return NextResponse.json(parsed.body, { status: parsed.status });
|
||||
const guard = assertTenantMatchesVerified(request, parsed.data.tenant_id);
|
||||
if (guard) return guard;
|
||||
return NextResponse.json(
|
||||
await sourceRegistryService.setAuthorization({ ...parsed.data, source_id: sourceId }),
|
||||
);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
/**
|
||||
* app/api/0pt/sources/[sourceId]/freshness/route.ts
|
||||
* =====================================================
|
||||
* Freshness view derived from provenance's freshness_sla_hours + the last
|
||||
* successful observation (recorded via POST .../health with status "up",
|
||||
* or SourceRegistryService.recordSuccess directly). Read-only — there is no
|
||||
* separate write path here by design; freshness is a derived signal, not
|
||||
* its own record.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { sourceRegistryService } from "@/src/services/source-registry";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const freshness = await sourceRegistryService.getFreshness(sourceId, tenantId);
|
||||
return NextResponse.json(freshness);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* app/api/0pt/sources/[sourceId]/health/route.ts
|
||||
* ==================================================
|
||||
* source.health — record a health observation (POST) and read the
|
||||
* aggregate view (GET): status, last check, consecutive failures. Bounded
|
||||
* history (latest 100 observations) — this is an operational signal, not
|
||||
* an audit log; AUD0 owns history.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, recordSourceHealthCheckSchema } from "@/src/validation/0pt-schemas";
|
||||
import { sourceRegistryService } from "@/src/services/source-registry";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const health = await sourceRegistryService.getHealth(sourceId, tenantId);
|
||||
return NextResponse.json(health);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const __raw = await request.json();
|
||||
const __parsed = parseBody(recordSourceHealthCheckSchema, __raw);
|
||||
if (!__parsed.ok) {
|
||||
return NextResponse.json(__parsed.body, { status: __parsed.status });
|
||||
}
|
||||
const body = __parsed.data;
|
||||
const tenantGuard = assertTenantMatchesVerified(request, body.tenant_id);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const health = await sourceRegistryService.recordHealthCheck({
|
||||
source_id: sourceId,
|
||||
tenant_id: body.tenant_id,
|
||||
status: body.status,
|
||||
checked_at: body.checked_at ?? new Date().toISOString(),
|
||||
latency_ms: body.latency_ms,
|
||||
detail: body.detail,
|
||||
});
|
||||
return NextResponse.json(health, { status: 201 });
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* app/api/0pt/sources/[sourceId]/provenance/route.ts
|
||||
* ======================================================
|
||||
* Per-SOURCE provenance/method metadata (Master-DoD §10) — HOW a source is
|
||||
* obtained (official_api | bulk_download | authorized_web | licensed_feed |
|
||||
* manual_upload), not per-field bridge_provenance. See BEFORE_STATE.md §3
|
||||
* for that distinction: this route does not, and cannot, close the
|
||||
* bridge_provenance gap — it only surfaces what SourceRegistryService
|
||||
* already models at the source level.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, setSourceProvenanceSchema } from "@/src/validation/0pt-schemas";
|
||||
import { sourceRegistryService } from "@/src/services/source-registry";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const provenance = await sourceRegistryService.getProvenance(sourceId, tenantId);
|
||||
if (!provenance) {
|
||||
return NextResponse.json({ error: `source ${sourceId} has no provenance record` }, { status: 404 });
|
||||
}
|
||||
return NextResponse.json(provenance);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function PUT(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const __raw = await request.json();
|
||||
const __parsed = parseBody(setSourceProvenanceSchema, __raw);
|
||||
if (!__parsed.ok) {
|
||||
return NextResponse.json(__parsed.body, { status: __parsed.status });
|
||||
}
|
||||
const body = __parsed.data;
|
||||
const tenantGuard = assertTenantMatchesVerified(request, body.tenant_id);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const provenance = await sourceRegistryService.setProvenance({
|
||||
...body,
|
||||
source_id: sourceId,
|
||||
updated_at: new Date().toISOString(),
|
||||
});
|
||||
return NextResponse.json(provenance);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -4,27 +4,25 @@
|
||||
* 0pt.query_source — bounded synchronous query execution for a registered
|
||||
* Source. Additive: does not modify sibling ../route.ts or ../../route.ts.
|
||||
*
|
||||
* NOTE — in-memory scope: this module instantiates its own SourceService,
|
||||
* matching the existing sources/route.ts and sources/[sourceId]/route.ts
|
||||
* pattern (each API route module owns a fresh in-memory registry, so state
|
||||
* is not shared across route modules yet — see the "Singleton note" in
|
||||
* src/services/document-capture-service.ts for prior art on this same repo
|
||||
* limitation). A source registered via POST /api/0pt/sources will not be
|
||||
* resolvable here until SourceService is backed by shared persistence.
|
||||
* Correction: this comment previously claimed `sql/002_p0_dataset_quality.sql`
|
||||
* already defined an `opt_sources` table for that. That file does not exist
|
||||
* and never has (git log --all: zero commits, any branch, ever). sql/ today
|
||||
* has only 001_init.sql and 004_iceberg_write_boundary.sql. The real fix —
|
||||
* backing SourceService with real persistence — is still open; no migration
|
||||
* for it exists yet.
|
||||
* Details: docs/P0_QUERY_SOURCE_FLOW.md.
|
||||
* UPDATED (Source Registry Postgres-persistence pass): now imports the
|
||||
* shared `sourceService` singleton from src/services/source-registry.ts
|
||||
* instead of constructing its own SourceService, closing the "not even a
|
||||
* shared process-wide singleton" gap BEFORE_STATE.md §1 flagged (a source
|
||||
* registered via POST /api/0pt/sources used to be invisible here). With
|
||||
* DATABASE_URL configured this is doubly fixed — state lives in Postgres
|
||||
* (sql/002_source_registry.sql), genuinely shared across processes, not
|
||||
* just within one. Details: docs/P0_QUERY_SOURCE_FLOW.md.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, sourceQuerySchema } from "@/src/validation/0pt-schemas";
|
||||
import { SourceService } from "@/src/services/source-service";
|
||||
import { SourceQueryService, SourceQueryTimeoutError, SourceQueryUnsupportedError } from "@/src/services/source-query-service";
|
||||
import { sourceService } from "@/src/services/source-registry";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
import {
|
||||
SourceQueryService,
|
||||
SourceQueryTimeoutError,
|
||||
SourceQueryUnsupportedError,
|
||||
} from "@/src/services/source-query-service";
|
||||
|
||||
const sourceService = new SourceService();
|
||||
const queryService = new SourceQueryService({ sourceService });
|
||||
|
||||
export async function POST(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
@@ -46,6 +44,8 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
|
||||
if (!body.connection_config || typeof body.connection_config !== "object") {
|
||||
return NextResponse.json({ error: "connection_config is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, body.tenant_id);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
|
||||
const result = await queryService.querySource({
|
||||
source_id: sourceId,
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* app/api/0pt/sources/[sourceId]/rights/route.ts
|
||||
* ==================================================
|
||||
* Rights / license / purpose metadata (Master-DoD §10). PUT is an upsert
|
||||
* (SourceRegistryService.setRights), matching the fail-closed purpose gate:
|
||||
* no rights record at all denies every purpose until one is set here.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, setSourceRightsSchema } from "@/src/validation/0pt-schemas";
|
||||
import { sourceRegistryService } from "@/src/services/source-registry";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const rights = await sourceRegistryService.getRights(sourceId, tenantId);
|
||||
if (!rights) {
|
||||
return NextResponse.json({ error: `source ${sourceId} has no rights record` }, { status: 404 });
|
||||
}
|
||||
return NextResponse.json(rights);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function PUT(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const __raw = await request.json();
|
||||
const __parsed = parseBody(setSourceRightsSchema, __raw);
|
||||
if (!__parsed.ok) {
|
||||
return NextResponse.json(__parsed.body, { status: __parsed.status });
|
||||
}
|
||||
const body = __parsed.data;
|
||||
const tenantGuard = assertTenantMatchesVerified(request, body.tenant_id);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const rights = await sourceRegistryService.setRights({ ...body, source_id: sourceId });
|
||||
return NextResponse.json(rights);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
/**
|
||||
* app/api/0pt/sources/[sourceId]/route.ts
|
||||
* ==========================================
|
||||
* source.describe — full SourceDescriptor (base + rights + provenance +
|
||||
* health + freshness + rate policy). Cross-tenant probes get 404 NOT_FOUND
|
||||
* (no existence oracle, §50) via SourceRegistryService.describeSource's
|
||||
* assertSourceTenant gate.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { sourceRegistryService } from "@/src/services/source-registry";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const descriptor = await sourceRegistryService.describeSource(sourceId, tenantId);
|
||||
return NextResponse.json(descriptor);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
/**
|
||||
* app/api/0pt/sources/[sourceId]/schedule/route.ts
|
||||
* =================================================
|
||||
* P060 "schedule": ingestion mode + cadence, recorded on the source. 0pt does
|
||||
* NOT run cron itself — ACT0 owns scheduling; this row is the declaration a
|
||||
* scheduler reads, and saying otherwise would be a claim this repo cannot back.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, setSourceScheduleSchema } from "@/src/validation/0pt-schemas";
|
||||
import { sourceRegistryService } from "@/src/services/source-registry";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
export async function GET(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const tenantId = new URL(request.url).searchParams.get("tenant_id");
|
||||
if (!tenantId) return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
const guard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (guard) return guard;
|
||||
const schedule = await sourceRegistryService.getSchedule(sourceId, tenantId);
|
||||
if (!schedule) {
|
||||
return NextResponse.json({ error: `source ${sourceId} has no schedule record` }, { status: 404 });
|
||||
}
|
||||
return NextResponse.json(schedule);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function PUT(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const parsed = parseBody(setSourceScheduleSchema, await request.json());
|
||||
if (!parsed.ok) return NextResponse.json(parsed.body, { status: parsed.status });
|
||||
const guard = assertTenantMatchesVerified(request, parsed.data.tenant_id);
|
||||
if (guard) return guard;
|
||||
return NextResponse.json(
|
||||
await sourceRegistryService.setSchedule({
|
||||
...parsed.data,
|
||||
source_id: sourceId,
|
||||
updated_at: new Date().toISOString(),
|
||||
}),
|
||||
);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,29 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { SourceRatePolicyService } from "@/src/services/source-rate-policy-service";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { sourceRatePolicyService as service } from "@/src/services/source-registry";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
const service = new SourceRatePolicyService();
|
||||
|
||||
export async function GET(_request: Request, { params }: { params: Promise<{ sourceRatePolicyId: string }> }) {
|
||||
// P014: this lookup used to accept only `sourceRatePolicyId`, with NO tenant
|
||||
// check anywhere in the request path — any authenticated caller could read
|
||||
// another tenant's rate-limit/crawl-budget/compliance-notes record by
|
||||
// guessing or enumerating IDs. Now requires `?tenant_id=` (matching every
|
||||
// sibling route under app/api/0pt/sources/*) and both checks it against the
|
||||
// verified token (assertTenantMatchesVerified) and threads it into the
|
||||
// service so a foreign policy_id 404s instead of leaking (no existence
|
||||
// oracle) — see getSourceRatePolicy's doc comment.
|
||||
export async function GET(
|
||||
request: NextRequest,
|
||||
{ params }: { params: Promise<{ sourceRatePolicyId: string }> },
|
||||
) {
|
||||
try {
|
||||
const { sourceRatePolicyId } = await params;
|
||||
const policy = await service.getSourceRatePolicy(sourceRatePolicyId);
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const policy = await service.getSourceRatePolicy(sourceRatePolicyId, tenantId);
|
||||
return NextResponse.json(policy);
|
||||
} catch (error: unknown) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, createSourceRatePolicySchema } from "@/src/validation/0pt-schemas";
|
||||
import { SourceRatePolicyService } from "@/src/services/source-rate-policy-service";
|
||||
|
||||
const service = new SourceRatePolicyService();
|
||||
import { sourceRatePolicyService as service } from "@/src/services/source-registry";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
@@ -12,6 +11,8 @@ export async function POST(request: NextRequest) {
|
||||
return NextResponse.json(__parsed.body, { status: __parsed.status });
|
||||
}
|
||||
const body = __parsed.data;
|
||||
const tenantGuard = assertTenantMatchesVerified(request, body.tenant_id);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const policy = await service.createSourceRatePolicy(body);
|
||||
return NextResponse.json(policy, { status: 201 });
|
||||
} catch (error: unknown) {
|
||||
@@ -22,8 +23,11 @@ export async function POST(request: NextRequest) {
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id") ?? undefined;
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const policies = await service.listSourceRatePolicies({
|
||||
tenant_id: searchParams.get("tenant_id") ?? undefined,
|
||||
tenant_id: tenantId,
|
||||
project_id: searchParams.get("project_id") ?? undefined,
|
||||
source_id: searchParams.get("source_id") ?? undefined,
|
||||
});
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* app/api/0pt/sources/route.ts
|
||||
* ==============================
|
||||
* source.register (POST) + source.query (GET) — the P0 Source Registry
|
||||
* route surface BEFORE_STATE.md §1 found entirely missing
|
||||
* (`grep -rl "SourceRegistryService" app/` returned nothing). Backed by
|
||||
* SourceRegistryService, which now persists to Postgres when DATABASE_URL
|
||||
* is configured (src/repositories/*, sql/002_source_registry.sql).
|
||||
*
|
||||
* Uses the shared singleton from src/services/source-registry.ts, not a
|
||||
* fresh `new SourceRegistryService()` per route module — see that file's
|
||||
* doc comment.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { parseBody, registerSourceSchema } from "@/src/validation/0pt-schemas";
|
||||
import { sourceRegistryService } from "@/src/services/source-registry";
|
||||
import { siaxErrorResponse } from "@/src/http/siax-error-response";
|
||||
import { assertTenantMatchesVerified } from "@/src/http/tenant-guard";
|
||||
import type { SourceStatus, SourceType } from "@/src/types/source";
|
||||
import type { SourceRights } from "@/src/types/source-registry";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const __raw = await request.json();
|
||||
const __parsed = parseBody(registerSourceSchema, __raw);
|
||||
if (!__parsed.ok) {
|
||||
return NextResponse.json(__parsed.body, { status: __parsed.status });
|
||||
}
|
||||
const body = __parsed.data;
|
||||
const tenantGuard = assertTenantMatchesVerified(request, body.tenant_id);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const descriptor = await sourceRegistryService.registerSource(body);
|
||||
return NextResponse.json(descriptor, { status: 201 });
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id");
|
||||
if (!tenantId) {
|
||||
return NextResponse.json({ error: "tenant_id is required" }, { status: 400 });
|
||||
}
|
||||
const tenantGuard = assertTenantMatchesVerified(request, tenantId);
|
||||
if (tenantGuard) return tenantGuard;
|
||||
const sources = await sourceRegistryService.querySources({
|
||||
tenant_id: tenantId,
|
||||
project_id: searchParams.get("project_id") ?? undefined,
|
||||
source_type: (searchParams.get("source_type") as SourceType) ?? undefined,
|
||||
status: (searchParams.get("status") as SourceStatus) ?? undefined,
|
||||
purpose: searchParams.get("purpose") ?? undefined,
|
||||
redistribution: (searchParams.get("redistribution") as SourceRights["redistribution"]) ?? undefined,
|
||||
});
|
||||
return NextResponse.json(sources);
|
||||
} catch (error: unknown) {
|
||||
return siaxErrorResponse(error);
|
||||
}
|
||||
}
|
||||
+35
-4
@@ -1,12 +1,43 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { checkAllDependencies, hasDegradedDependency } from "@/lib/health-checks";
|
||||
|
||||
// GET /api/health — platform liveness (docs/04). DB/queue wired when provisioned.
|
||||
// GET /api/health — liveness (docs/04, BACKLOG P0-3).
|
||||
//
|
||||
// Fixed finding: this used to hardcode `ok: true` and only ever check
|
||||
// *presence* of DATABASE_URL/REDIS_URL, never whether anything was actually
|
||||
// reachable -- "an uptime monitor watching HTTP 200 would show green for a
|
||||
// dying instance" (docs/architecture/ARCHITECTURE.md). Each dependency below
|
||||
// now reports a real state: "ok" (live round trip succeeded), "error"
|
||||
// (configured but unreachable -- see `detail`), "not_configured", or
|
||||
// "not_integrated" (env var present but no client code reads it -- honest
|
||||
// about `queue`, which has no wired Redis client in this repo).
|
||||
//
|
||||
// This route stays a LIVENESS check: it always answers 200 as long as the
|
||||
// process itself can run the handler (that's what "liveness" means -- an
|
||||
// external dependency outage is not a reason to restart this instance). The
|
||||
// body's `ok` field is the honest signal instead: false whenever a
|
||||
// dependency that IS configured is failing. For a readiness gate that DOES
|
||||
// fail closed on missing/broken dependencies, see GET /api/ready.
|
||||
export async function GET() {
|
||||
const deps = await checkAllDependencies();
|
||||
const degraded = hasDegradedDependency(deps);
|
||||
|
||||
return NextResponse.json({
|
||||
ok: true,
|
||||
ok: !degraded,
|
||||
service: "0pt-web",
|
||||
db: Boolean(process.env.DATABASE_URL) ? "configured" : "not_configured",
|
||||
queue: Boolean(process.env.REDIS_URL) ? "configured" : "not_configured",
|
||||
db: deps.db.status,
|
||||
queue: deps.queue.status,
|
||||
engine: deps.engine.status,
|
||||
stripe: deps.stripe.status,
|
||||
...(degraded
|
||||
? {
|
||||
detail: Object.fromEntries(
|
||||
Object.entries(deps)
|
||||
.filter(([, v]) => v.status === "error")
|
||||
.map(([k, v]) => [k, v.detail]),
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
ts: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
+22
-3
@@ -1,11 +1,19 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { createOrder, listOrders, type OrderConfig } from "@/lib/orders-store";
|
||||
import { createOrder, listOrders, IdempotencyKeyConflictError, type OrderConfig } from "@/lib/orders-store";
|
||||
import { requireRouteAuth } from "@/src/auth/require-route-auth";
|
||||
|
||||
// POST /api/orders (docs/04) — create a draft order from a configurator config.
|
||||
// Intentionally anonymous: a prospect drafts/estimates a job before signing in
|
||||
// to pay the deposit (see docs/01, the configurator flow). Only reading back
|
||||
// the order queue (GET, below) is a data-exposure risk.
|
||||
//
|
||||
// P014: honors an optional `Idempotency-Key` header (the convention this
|
||||
// skill's own docs already name -- "Idempotency-Key on order/payment POSTs")
|
||||
// so a retried submission (double-click, dropped response, proxy replay)
|
||||
// returns the ORIGINAL order (200) instead of creating a duplicate (was:
|
||||
// no idempotency mechanism at all -- every retry created a new row, P014
|
||||
// recon "Duplicate delivery idempotent: NOT MET"). Omitting the header keeps
|
||||
// the pre-P014 behavior exactly: every call creates a fresh order.
|
||||
export async function POST(req: NextRequest) {
|
||||
let body: { flow?: string; config?: OrderConfig };
|
||||
try {
|
||||
@@ -18,8 +26,19 @@ export async function POST(req: NextRequest) {
|
||||
}
|
||||
const flow = body.flow || "custom";
|
||||
const config = body.config || {};
|
||||
const order = await createOrder(flow, config);
|
||||
return NextResponse.json({ order_id: order.order_id, status: order.status, deposit_ore: order.deposit_ore, total_ore: order.total_ore }, { status: 201 });
|
||||
const idempotencyKey = req.headers.get("idempotency-key") ?? undefined;
|
||||
try {
|
||||
const { order, replayed } = await createOrder(flow, config, idempotencyKey);
|
||||
return NextResponse.json(
|
||||
{ order_id: order.order_id, status: order.status, deposit_ore: order.deposit_ore, total_ore: order.total_ore },
|
||||
{ status: replayed ? 200 : 201 },
|
||||
);
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof IdempotencyKeyConflictError) {
|
||||
return NextResponse.json({ error: { code: "idempotency_key_conflict", message: err.message } }, { status: 409 });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// GET /api/orders — list all orders (queue + financial aggregates). This is
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { checkAllDependencies, isReady } from "@/lib/health-checks";
|
||||
|
||||
// GET /api/ready — readiness (BACKLOG P0-3, CD-04/OBS-04: "no /ready route
|
||||
// existed at all").
|
||||
//
|
||||
// Unlike GET /api/health (liveness -- always 200 while the process can
|
||||
// respond), this fails closed: 503 whenever a load-bearing dependency isn't
|
||||
// both configured AND actually reachable. Point a load balancer / Coolify
|
||||
// readiness probe at this route, not /api/health, to stop routing traffic to
|
||||
// an instance whose database is down without restarting it.
|
||||
//
|
||||
// Today the only load-bearing dependency is the database (see
|
||||
// `lib/health-checks.ts#isReady`) -- engine/stripe/queue are optional
|
||||
// integrations 0pt is designed to run without.
|
||||
export async function GET() {
|
||||
const deps = await checkAllDependencies();
|
||||
const ready = isReady(deps);
|
||||
|
||||
return NextResponse.json(
|
||||
{
|
||||
ready,
|
||||
service: "0pt-web",
|
||||
db: deps.db.status,
|
||||
queue: deps.queue.status,
|
||||
engine: deps.engine.status,
|
||||
stripe: deps.stripe.status,
|
||||
ts: new Date().toISOString(),
|
||||
},
|
||||
{ status: ready ? 200 : 503 },
|
||||
);
|
||||
}
|
||||
+47
-9
@@ -1,10 +1,48 @@
|
||||
# DX-02 — lokala beroenden för 0PT.
|
||||
# BACKLOG P1-19 (DX-02): local-only Postgres for developers who want real
|
||||
# persistence instead of the in-memory fallback (lib/db.ts / lib/orders-store.ts
|
||||
# already degrade gracefully without this -- `pnpm dev` / `make dev` work
|
||||
# without it, this is additive). Modeled on the already-proven Postgres
|
||||
# service in infra/docker-compose.yml (same image, same sql/ init mount) --
|
||||
# not a copy of the full VPS stack (no NocoDB/Metabase/Caddy/worker here,
|
||||
# those are production concerns, not local dev needs).
|
||||
#
|
||||
# TODO: deklarera repots FAKTISKA lokala beroenden (databas, cache, objektlager,
|
||||
# meddelandebuss …) så att DX-01 håller: ett kommando ska resa allt lokalt, utan
|
||||
# åtkomst till produktion.
|
||||
#
|
||||
# Tomt värde nedan betyder "repot har inga lokala beroenden". Scaffolden kan
|
||||
# inte veta om det är sant — den lämnar det tomt i stället för att hitta på
|
||||
# tjänster som ingen kör.
|
||||
services: {}
|
||||
# VERIFIED (P014, 2026-09-03): this file's image/credentials/init-mount shape
|
||||
# was exercised for real (an equivalent standalone container -- this sandbox
|
||||
# had an unrelated service already bound to host port 5432, so the literal
|
||||
# `docker compose -f compose.dev.yaml up` command itself could not be run
|
||||
# here, but the identical image + `sql/*.sql` init sequence + connection
|
||||
# shape were run and verified end to end as part of P014's restore drill).
|
||||
# That drill surfaced and fixed the ACTUAL reason this was unverifiable
|
||||
# before: `lib/db.ts`'s `getSql()` hardcoded `ssl: "require"`, which this
|
||||
# plain-Postgres container can never satisfy (no TLS is configured here) --
|
||||
# every connection attempt failed before a single query ran. Fixed: `ssl` now
|
||||
# defaults to "require" (unchanged for every real deployment) but honors
|
||||
# `DATABASE_SSL=false` to opt out. To use this container from the app or a
|
||||
# script, set BOTH:
|
||||
# DATABASE_URL=postgresql://opt:opt-dev-only-not-a-secret@localhost:5432/opt
|
||||
# DATABASE_SSL=false
|
||||
# `sql/003_datasets.sql` (also added by P014, closing a separate gap where
|
||||
# `004_iceberg_write_boundary.sql` referenced a table no migration defined)
|
||||
# is picked up by the `docker-entrypoint-initdb.d` mount automatically --
|
||||
# Postgres runs `*.sql` in this directory in filename-sorted order, and the
|
||||
# zero-padded 00N prefixes already sort correctly.
|
||||
services:
|
||||
db:
|
||||
image: pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: opt
|
||||
POSTGRES_PASSWORD: opt-dev-only-not-a-secret
|
||||
POSTGRES_DB: opt
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- opt-dev-pgdata:/var/lib/postgresql/data
|
||||
- ./sql:/docker-entrypoint-initdb.d:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U opt -d opt"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
volumes:
|
||||
opt-dev-pgdata: {}
|
||||
|
||||
+75
-5
@@ -22,15 +22,35 @@ paths:
|
||||
/api/health:
|
||||
get:
|
||||
operationId: 0pt.health
|
||||
summary: Liveness + dependency state
|
||||
summary: Liveness + real per-dependency state
|
||||
x-siax-status: live
|
||||
security: []
|
||||
responses:
|
||||
"200":
|
||||
description: OK — db/queue rapporterar "configured"/"not_configured"
|
||||
description: Process is alive. Body reports honest per-dependency status (ok/error/not_configured/not_integrated) — see HealthResponse.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/HealthResponse"
|
||||
/api/ready:
|
||||
get:
|
||||
operationId: 0pt.ready
|
||||
summary: Readiness — fails closed when a load-bearing dependency is down
|
||||
x-siax-status: live
|
||||
security: []
|
||||
responses:
|
||||
"200":
|
||||
description: Ready to serve real traffic.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ReadyResponse"
|
||||
"503":
|
||||
description: Not ready — a load-bearing dependency is not configured/reachable.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ReadyResponse"
|
||||
/api/estimates:
|
||||
post:
|
||||
operationId: 0pt.estimate_order
|
||||
@@ -401,7 +421,18 @@ components:
|
||||
type: string
|
||||
HealthResponse:
|
||||
type: object
|
||||
required: [ok, service, db, queue, ts]
|
||||
description: >
|
||||
db/engine/stripe report a real dependency probe (BACKLOG P0-3): "ok"
|
||||
means a live round trip succeeded, "error" means it was configured
|
||||
but unreachable (see `detail`), "not_configured" means the env var
|
||||
is unset. `queue` can only ever be "not_configured" or
|
||||
"not_integrated" -- no Redis client is wired in this repo, so
|
||||
presence of REDIS_URL is reported honestly rather than faking a
|
||||
connectivity check. `ok` is true only when no configured dependency
|
||||
is in "error". This is a LIVENESS check: HTTP status stays 200
|
||||
whenever the process can respond, regardless of dependency state --
|
||||
see /api/ready for the readiness gate that fails closed.
|
||||
required: [ok, service, db, queue, engine, stripe, ts]
|
||||
properties:
|
||||
ok:
|
||||
type: boolean
|
||||
@@ -409,10 +440,49 @@ components:
|
||||
type: string
|
||||
db:
|
||||
type: string
|
||||
enum: [configured, not_configured]
|
||||
enum: [ok, error, not_configured]
|
||||
queue:
|
||||
type: string
|
||||
enum: [configured, not_configured]
|
||||
enum: [not_configured, not_integrated]
|
||||
engine:
|
||||
type: string
|
||||
enum: [ok, error, not_configured]
|
||||
stripe:
|
||||
type: string
|
||||
enum: [ok, error, not_configured]
|
||||
detail:
|
||||
type: object
|
||||
description: Present only when `ok` is false -- one-line error message per failing dependency.
|
||||
additionalProperties:
|
||||
type: string
|
||||
ts:
|
||||
type: string
|
||||
format: date-time
|
||||
ReadyResponse:
|
||||
type: object
|
||||
description: >
|
||||
Readiness (BACKLOG P0-3: "no /ready route existed"). Fails closed --
|
||||
503 whenever a load-bearing dependency (today: only `db`) is not
|
||||
both configured and reachable. Point a load balancer / Coolify
|
||||
readiness probe here, not at /api/health.
|
||||
required: [ready, service, db, queue, engine, stripe, ts]
|
||||
properties:
|
||||
ready:
|
||||
type: boolean
|
||||
service:
|
||||
type: string
|
||||
db:
|
||||
type: string
|
||||
enum: [ok, error, not_configured]
|
||||
queue:
|
||||
type: string
|
||||
enum: [not_configured, not_integrated]
|
||||
engine:
|
||||
type: string
|
||||
enum: [ok, error, not_configured]
|
||||
stripe:
|
||||
type: string
|
||||
enum: [ok, error, not_configured]
|
||||
ts:
|
||||
type: string
|
||||
format: date-time
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# Kubernetes manifests — reference only, NOT the deploy path
|
||||
|
||||
BACKLOG P6 (CD-23). **0pt's actual deploy target is Coolify** (ADR-0001,
|
||||
`siax.config.json`), which runs the committed `Dockerfile` directly — it
|
||||
does not consume Kubernetes manifests. `deploy/k8s/deployment.yaml` in this
|
||||
directory is a reference example only, for the scenario where a future SIAX
|
||||
Cloud node runs k8s instead of/alongside Coolify. It is not applied by any
|
||||
CI/CD path in this repo, and nothing in `.gitea/workflows/` references it.
|
||||
|
||||
Do not `kubectl apply` this against a real cluster without first replacing
|
||||
the placeholder image reference and reviewing resource limits against actual
|
||||
measured usage (none exists yet — 0pt has no live traffic history to size
|
||||
against, see BACKLOG P0-2/D5).
|
||||
@@ -0,0 +1,63 @@
|
||||
# Reference only -- see deploy/k8s/README.md. Not applied by any CI/CD path.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: 0pt-web
|
||||
labels:
|
||||
app: 0pt-web
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app: 0pt-web
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: 0pt-web
|
||||
spec:
|
||||
containers:
|
||||
- name: 0pt-web
|
||||
image: REPLACE_ME/0pt-web:REPLACE_ME_TAG
|
||||
ports:
|
||||
- containerPort: 3000
|
||||
env:
|
||||
- name: NODE_ENV
|
||||
value: "production"
|
||||
# Real values come from Infisical (BACKLOG P0-1), never from a
|
||||
# committed manifest -- see siax.config.json's `env` list for the
|
||||
# names this app reads.
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: 0pt-web-env
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /api/health
|
||||
port: 3000
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 15
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /api/ready
|
||||
port: 3000
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
# Placeholder -- no live traffic history exists to size against
|
||||
# (BACKLOG P0-2/D5). Replace before any real use.
|
||||
requests:
|
||||
cpu: "250m"
|
||||
memory: "256Mi"
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: "512Mi"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: 0pt-web
|
||||
spec:
|
||||
selector:
|
||||
app: 0pt-web
|
||||
ports:
|
||||
- port: 80
|
||||
targetPort: 3000
|
||||
+71
-7
@@ -37,7 +37,7 @@ This principle describes — and is largely realized by — the **original edge
|
||||
- **FastAPI orchestrator** (`services/api/orchestrator/main.py`) — REST contract in `docs/04-api.md`: estimates, orders, sources, vehicle lookup, sanctions screening, job run/status. **Not on Vercel** (serverless timeouts kill long jobs). *Correction: earlier drafts of this doc pointed at `src/orchestrator`; that was the pre-migration location. `docs/09-build-plan.md` Phase 0 called for moving it into `services/api`, and that move has happened — the code lives at `services/api/orchestrator` today.*
|
||||
- **Workers** (`services/api/workers/`) — adaptive router (`adaptive_router.py`, verified: chooses only among compliant methods — official API → bulk → authorized web → licensed, no bypass/stealth/proxy-rotation tier exists in it, by design) + pipeline stages (`pipeline.py`, `queue.py`, `resolve.py`, `run.py`). See `docs/06-data-engine.md`.
|
||||
- **Postgres** — backbone (`docs/03-data-model.md`) incl. `pgvector` for matching/embeddings, for the original edge + engine plane. Wired via `lib/db.ts` (env-guarded: `DATABASE_URL`/`POSTGRES_URL`; falls back to an in-memory, non-durable store when unset, e.g. `lib/orders-store.ts`).
|
||||
- **NocoDB / Metabase** — both are defined as services in `infra/docker-compose.yml`; embedding them into the portal as "Min data" / customer dashboards is the target, not yet confirmed wired into a portal page in this pass.
|
||||
- **NocoDB / Metabase** — both are defined as services in `infra/docker-compose.yml`; embedding them into the portal as "Min data" / customer dashboards was the original target, and neither is confirmed wired into a portal page. *Correction 2026-09-06: Metabase is no longer the canonical BI lens — Apache Superset is (see "Stack decisions" below). Metabase may stay as an optional App Catalog app; it must not be treated as the platform default.*
|
||||
- **Cloudflare R2** — file storage (uploads + deliverables) via presigned URLs. Env-guarded (`app/api/uploads/route.ts` checks `R2_ACCESS_KEY_ID`/`R2_BUCKET`/`R2_ENDPOINT` and returns `501` until set); the presign call itself is still a stub (`put_url: null`) even when the env vars are present — not wired end-to-end yet.
|
||||
- **Stripe** — deposit capture + escrow balance release. This one **is** wired: `app/api/payments/[order_id]/deposit/route.ts` and `.../release/route.ts` are real Stripe Checkout calls (env-guarded on `STRIPE_SECRET_KEY`, `501` until configured). See the correction in `docs/01-product.md`'s business model section for exactly which escrow mechanism is implemented.
|
||||
- **Resend** — documented as the transactional-email provider (`docs/08-infrastructure.md`) but has no call site anywhere in this repo yet. Treat "+ email" as not-yet-implemented, not merely unconfigured.
|
||||
@@ -101,7 +101,21 @@ In plain terms: **0pt is API-only today** (plus two job-backed writes, and one j
|
||||
|
||||
`src/clients/{cl0ud,n0d,aud0,st0re,st0re-lakehouse}-client.ts` are real, typechecked TypeScript with accompanying tests (`tests/p0/*`), each following the same shape: a thin interface, an HTTP implementation written against the target service's real, committed route contract, and a deterministic in-memory mock used whenever no `baseUrl` is supplied to the constructor.
|
||||
|
||||
**Verified this pass**: none of the five client files, and none of their call sites in `src/services/*`, read `CL0UD_URL`, `N0D_URL`, `AUD0_URL`, or `ST0RE_URL` from `process.env` anywhere (`grep -rn "process.env" src/clients src/services` turns up nothing for any of those four names outside comments/TODOs). Every constructor call in this repo (`createCl0udClient(opts?.cl0udBaseUrl)`, etc.) is invoked with no `baseUrl` argument, so **every one of these clients runs against its in-memory mock today, in every code path that exists in this repo.** Read this as "real code, mock-only, not live-verified" — not "integrated" or "connected" — wherever platform-client status comes up.
|
||||
**SUPERSEDED 2026-08-31 (SIAX Platforms P0 wave 2)** — the paragraph that
|
||||
stood here ("every one of these clients runs against its in-memory mock
|
||||
today") was accurate when written and is now fixed, twice over:
|
||||
|
||||
1. every factory reads its `*_URL` from `process.env` when no constructor
|
||||
`baseUrl` is given, and under `NODE_ENV=production` an unset URL is a hard
|
||||
boot failure (`SiaxError DEPENDENCY_UNAVAILABLE`, `src/platform/dependency-mode.ts`)
|
||||
— production can no longer fall back to a mock (Master-DoD §3/§10);
|
||||
2. the HTTP wire shapes were verified against the sibling repos' actual
|
||||
`main` branches and THREE of four were wrong (cl0ud path never existed,
|
||||
st0re body was snake_case against a camelCase contract, aud0 lineage path
|
||||
never existed). All rewritten and pinned by `tests/contracts/*.contract.test.ts`.
|
||||
|
||||
Current per-capability status lives in [`STATUS.md`](../STATUS.md) — that
|
||||
file, not this doc, is the status source of truth (§56).
|
||||
|
||||
What each client is for, concretely:
|
||||
|
||||
@@ -143,7 +157,7 @@ Flagged so it is not repeated: `src/services/quality-check-service.ts`, `src/ser
|
||||
| Backend/jobs | FastAPI + workers (`services/api`) | already built; long-running jobs |
|
||||
| DB | Postgres (+ pgvector) | backbone; Neon free tier or on-VPS |
|
||||
| Airtable layer | **NocoDB** | sits on existing tables, no duplication |
|
||||
| BI | **Metabase** | customer-facing embedding, simple |
|
||||
| BI | **Apache Superset** | canonical generic BI for SIAX Cloud: Apache-2.0 with an official embed SDK + guest tokens, so embedding never needs a commercial tier. Already defined as a service in `services/infra/docker-compose.data-stack.yml` and inventoried in `docs/compliance/OSS-INVENTORY.md`. **Corrected 2026-09-06** — this row previously said Metabase; see the correction under "OUT" below |
|
||||
| Files | Cloudflare R2 + presigned URLs | cheapest S3-compatible |
|
||||
| Payments | Stripe Checkout + webhook → Next.js API | ~1 day, no SaaS boilerplate needed |
|
||||
| Auth | Auth.js + Postgres adapter | one system — **target, not yet built** (see Components above) |
|
||||
@@ -152,21 +166,71 @@ Flagged so it is not repeated: `src/services/quality-check-service.ts`, `src/ser
|
||||
|
||||
## Stack decisions — OUT (do not add)
|
||||
|
||||
> **CORRECTED 2026-09-06 — two rows in this table were wrong and are withdrawn below.**
|
||||
> This section was written when 0pt was planned as a standalone SaaS product. Under the SIAX
|
||||
> Cloud analytics decision, **0pt is the canonical Analytics/Data/Semantic owner** — there is
|
||||
> no `BI0`/`Analytics0` repo, no separate Superset platform, and no product-specific BI
|
||||
> backend acting as a parallel capability owner. Third-party BI tools are interchangeable
|
||||
> providers or App Catalog apps. Two consequences contradict the table as it stood:
|
||||
>
|
||||
> - **Apache Superset is canonical**, not rejected (Apache-2.0, official embed SDK + guest
|
||||
> tokens). Metabase moves to the optional App Catalog.
|
||||
> - **OpenMetadata is the canonical catalog** (default *before* DataHub), not rejected.
|
||||
> DataHub remains optional.
|
||||
>
|
||||
> This is not a paper change — **this repository already shipped both**, which is what makes
|
||||
> the old rows self-contradicting rather than merely out of date:
|
||||
>
|
||||
> - `src/metadata/openmetadata-adapter.ts` + `tests/contracts/metadata-features.contract.test.ts`
|
||||
> (OpenMetadata projection; `STATUS.md` classifies it `IMPLEMENTED`, projection-never-canonical),
|
||||
> - `services/infra/docker-compose.data-stack.yml` defines a `superset` service
|
||||
> (`apache/superset:latest`), and `docs/compliance/OSS-INVENTORY.md` already inventories
|
||||
> `superset | Apache-2.0 | services/infra/data-stack`.
|
||||
>
|
||||
> The governing principle behind the change is **one definition of every metric**: the 0pt
|
||||
> Metric Catalog feeds MetricFlow, which feeds Superset / Cube / the native API / INF0. Point a
|
||||
> product at a BI engine directly and each surface invents its own "average price".
|
||||
|
||||
| Rejected | Reason |
|
||||
|---|---|
|
||||
| Superset | Metabase already covers BI — one tool, not two |
|
||||
| DataHub / OpenMetadata | enterprise-heavy; catalog = a Postgres table in Next.js + NocoDB. Add lineage only if an enterprise customer demands it |
|
||||
| ~~Superset~~ | **WITHDRAWN 2026-09-06 — see the correction note above.** Superset is now the canonical BI engine, not a rejected one |
|
||||
| ~~DataHub / OpenMetadata~~ | **WITHDRAWN 2026-09-06 — see the correction note above.** OpenMetadata is the canonical catalog; only DataHub stays non-canonical (optional App Catalog) |
|
||||
| Metabase / Lightdash / Rill / Grafana / DataHub as *canonical* BI or catalog | optional App Catalog apps only, never the platform default. Metabase's OSS build is AGPL and its full app-/SSO-embedding sits behind Pro/Enterprise, which does not work for SIAX as a platform provider; Lightdash gates embedding/caching behind an enterprise self-host licence; Rill's documented deployment path is bound to Rill Cloud (breaks zero-SaaS); Grafana (AGPLv3) is the right tool for **operational** analysis and is AUD0-owned, not business BI |
|
||||
| Baserow / Teable | NocoDB chosen — not three Airtable clones |
|
||||
| Meilisearch (v1) | Postgres full-text suffices to a few thousand rows; add later |
|
||||
| Delta Sharing (v1) | unnecessary; customer pull via scoped role + exports |
|
||||
| Trigger.dev | the existing queue already schedules jobs; Resend covers mail |
|
||||
| FlareSolverr / stealth browsers / undetected-* / proxy-fingerprint rotation / fake-account creation | **prohibited** — see CLAUDE.md §1. Replace with compliant fetching. (The Gitea-only scraper stack described under "Two diverged git remotes" above is exactly this — removed there, never present on GitHub `main`.) |
|
||||
|
||||
## Hosting (cheapest coherent)
|
||||
## Hosting (cheapest coherent) — **SUPERSEDED** (BACKLOG P2-5)
|
||||
|
||||
> **SUPERSEDED 2026-08-11.** This section describes the original Vercel+Neon
|
||||
> hosting plan. `siax.config.json` flipped `deploy.target` from `"vercel"` to
|
||||
> `"coolify"` on 2026-08-09 to match every other SIAX Cloud OS repo — that
|
||||
> decision supersedes the Vercel prescription below. ~~The live instance is
|
||||
> **still actually running on Vercel today** (no Coolify app has been
|
||||
> provisioned yet — see `README.md`'s "Deploy" section and
|
||||
> `docs/architecture/ARCHITECTURE.md` §5/§8 for the current, verified state)~~;
|
||||
> this section is kept for history, not as the current target. Do not read
|
||||
> "Vercel (Hobby/free)" below as still-current guidance.
|
||||
>
|
||||
> **CORRECTED 2026-09-06 — the struck sentence above is no longer true.** The Coolify
|
||||
> deploy hook is live: repo secret `COOLIFY_TOKEN` exists (created 2026-08-31T11:34:34Z)
|
||||
> and repo Actions variable `COOLIFY_APP_ID` holds a real 24-character UUID, not the
|
||||
> `TODO-no-coolify-app-provisioned-yet` literal, so both guards in
|
||||
> `.gitea/workflows/cd.yml` pass. On the push to `main` at `1ff34410` (2026-09-06) the
|
||||
> `Deploy to Coolify / deploy` job finished **success** with `coolify deploy HTTP 200`.
|
||||
> **Every merge into `main` fires a real redeploy against cloud.siax.io.**
|
||||
>
|
||||
> Probed 2026-09-06 08:00 UTC: `0pt.app` still answers from **Vercel** (`server: Vercel`)
|
||||
> with **HTTP 402 `DEPLOYMENT_DISABLED`** on both `/` and `/api/health`. The public domain
|
||||
> is therefore *down*, DNS has not been cut over, and the Coolify app serves no `0pt.app`
|
||||
> traffic. Still unestablished: whether the Coolify app builds and starts at all — an
|
||||
> accepted deploy request is not a healthy container.
|
||||
|
||||
- **Vercel** (Hobby/free) — Next.js + custom domain `0pt.app`.
|
||||
- **One small VPS** (e.g. Hetzner ~€5/mo) running `docker-compose` (`infra/docker-compose.yml`): FastAPI, workers, NocoDB, Metabase, (Postgres or use Neon), optional Meilisearch later. Per-service managed platforms (Railway) get expensive once you have 4–5 always-on services; a single VPS + Vercel + R2 lands ≈ <€10/mo to start.
|
||||
- **Cloudflare R2** — object storage (env-guarded, presign not yet wired — see Components above).
|
||||
- **Postgres** — on the VPS, or **Neon** free tier if you want it managed.
|
||||
|
||||
Full config, env, security, CI/CD: `docs/08-infrastructure.md`. Env var inventory: `.env.example` exists on branch/PR [`feature/0pt-env-example`](https://github.com/sax3l/0pt/pull/3) (unmerged as of this pass) — until it merges to `main`, `docs/08-infrastructure.md` is the source of truth for env var names.
|
||||
Full config, env, security, CI/CD: `docs/08-infrastructure.md`. Env var inventory: [`.env.example`](../.env.example) (present in this repo today — the linked GitHub PR that originally tracked it has long since merged; removed the stale cross-repo link per BACKLOG P3-5).
|
||||
|
||||
@@ -74,6 +74,6 @@ sourcing and rapidfuzz-based dedup are real today; extraction/OCR, NER, and qual
|
||||
- **Bypass/stealth/evasion**: FlareSolverr, "CloakBrowser", undetected-chromedriver/undetected-playwright (as bypass), proxy/fingerprint-rotation engines, any "WAF bypass" repo. Prohibited by CLAUDE.md §1 — they destroy the honest-accuracy positioning and create legal exposure. **This is not hypothetical**: a real WAF-bypass/proxy-rotation/fake-identity scraper stack (`services/scrapers/*`) briefly existed on this repo's Gitea remote (`git.siax.io/sax3l/0pt`, three commits dated 2026-07-13) and has since been removed there via PR #1. It never touched `origin` (`github.com/sax3l/0pt`), which this doc corpus was verified against. See `docs/06-data-engine.md`'s Known Issue callout for the full, unresolved detail — which remote is canonical is Simon's call, not decided this pass.
|
||||
- **Fake-account automation** on Apollo/Cognism/etc.
|
||||
- **Unverified / likely-hallucinated** from earlier brainstorming: "CloakBrowser/CloakHQ", "Rota proxy engine", "OpenClaw". Do not adopt unless you confirm a real, maintained project — and even then, not if it falls in the bypass category.
|
||||
- **Overlapping tools** already decided against (Superset, DataHub/OpenMetadata, Baserow/Teable, Trigger.dev, Delta Sharing in v1) — see `docs/02-architecture.md`.
|
||||
- **Overlapping tools** already decided against (Baserow/Teable, Trigger.dev, Delta Sharing in v1) — see `docs/02-architecture.md`. *Corrected 2026-09-06: Superset and OpenMetadata were previously listed here as rejected. They are canonical (BI engine and catalog respectively) — see the correction note under "Stack decisions — OUT" in `docs/02-architecture.md`.*
|
||||
|
||||
> Note on real-but-overhyped tools (Firecrawl, browser-use, etc.): they exist, but ignore inflated star-counts and only adopt if they fit a *compliant* role and don't duplicate the stack.
|
||||
|
||||
@@ -137,12 +137,13 @@ dash.0pt.app { reverse_proxy metabase:3000 }
|
||||
|
||||
## Environment variables
|
||||
|
||||
**`.env.example` now exists** — added on branch `feature/0pt-env-example`
|
||||
(`github.com/sax3l/0pt/pull/3`, **not yet merged to `main`** as of this pass). It was previously
|
||||
impossible to commit: `.gitignore`'s blanket `.env*` rule silently matched `.env.example` too, so it
|
||||
was never added despite `docs/09-build-plan.md` Phase 0 requiring it since day one. The same PR adds
|
||||
`!.env.example` to `.gitignore` to fix that. README's "Env inventory: `.env.example`" link is
|
||||
therefore currently a broken/future reference on `main` until that PR merges.
|
||||
**`.env.example` now exists and is merged to `main`** (BACKLOG P3-5: the stale
|
||||
link to the unmerged `github.com/sax3l/0pt/pull/3` that used to sit here has
|
||||
been removed — that PR landed a while ago). It was previously impossible to
|
||||
commit: `.gitignore`'s blanket `.env*` rule silently matched `.env.example`
|
||||
too, so it was never added despite `docs/09-build-plan.md` Phase 0 requiring
|
||||
it since day one; `.gitignore` now has `!.env.example` to fix that. README's
|
||||
"Env inventory: `.env.example`" link is a real, current reference.
|
||||
|
||||
The table below is grep-verified against `main` this pass, not transcribed from `.env.example`'s own
|
||||
comments — in one case (`ST0RE_URL`, see note) that turned up a discrepancy worth flagging.
|
||||
|
||||
@@ -29,10 +29,10 @@ Phased, in dependency order. **Finish a phase to its acceptance criteria before
|
||||
**No `sql/002_*` or `sql/003_*` exists anywhere in this repo's history** — some other docs
|
||||
(`docs/ICEBERG_WRITE_BOUNDARY.md` etc.) cite a `sql/002_p0_dataset_quality.sql` that was never built;
|
||||
see `docs/08-infrastructure.md`'s "Schema state" section.
|
||||
- `.env.example` — **now exists**, on branch `feature/0pt-env-example`
|
||||
(`github.com/sax3l/0pt/pull/3`), **not yet merged to `main`**. It was blocked for the life of this
|
||||
project by `.gitignore`'s blanket `.env*` rule silently matching it too; the same PR fixes that
|
||||
with `!.env.example`. Until that PR merges, `main` still has no `.env.example`.
|
||||
- `.env.example` — **now exists and is merged to `main`** (BACKLOG P3-5: the
|
||||
stale link to the unmerged `github.com/sax3l/0pt/pull/3` that used to sit
|
||||
here has been removed — that PR has long since landed, `.env.example` is
|
||||
a real, current, gitignore-exempted file at the repo root today).
|
||||
- `infra/docker-compose.yml` and `infra/Caddyfile` are real, committed files (not sketches) covering
|
||||
exactly the services listed. **This pass found no evidence they've ever actually been run** — no
|
||||
CI, no deploy log, no reachable host checked. "File exists and looks correct" ≠ "verified to run";
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
# 0pt ingestion plane (P060–P068)
|
||||
|
||||
> Written 2026-09-04. Every claim here is backed by a test path or a
|
||||
> recorded command; anything not verified says so. Read `STATUS.md`'s
|
||||
> "P060-P068 ingestion plane" section for the classification table and the
|
||||
> honest-gaps list — this document is the mechanism, that one is the status.
|
||||
|
||||
## The shape
|
||||
|
||||
```
|
||||
Source Registry Connector Ingestion Runner
|
||||
(P060: who, what, (P061: nine (P062: one envelope,
|
||||
under what rights, operations, one one durable checkpoint)
|
||||
on what schedule) conformance suite)
|
||||
│ │ │
|
||||
│ classification gate │ discover / validate │ read ─┐
|
||||
│ blocks activation │ connect / read │ │
|
||||
▼ ▼ resume / backfill ▼ ▼
|
||||
opt_sources records ───────────────▶ IngestionEnvelope
|
||||
opt_source_rights │
|
||||
opt_source_authorization ├─▶ P063 schema gate
|
||||
opt_source_schedule ├─▶ P064 quality gate
|
||||
opt_source_schemas ├─▶ P068 ledger (peek)
|
||||
opt_ingestion_checkpoints ├─▶ SINK (caller-supplied)
|
||||
├─▶ P068 ledger (commit)
|
||||
├─▶ checkpoint (commit)
|
||||
└─▶ P067 lineage edge
|
||||
```
|
||||
|
||||
## P060 — the registry, and the gate
|
||||
|
||||
A source now carries: `source_id`, `tenant_id`, `project_id`, `source_type`,
|
||||
`owner_id`, an **authorization** record, a **license/purpose rights** record,
|
||||
**provenance** (how the data is obtained), a **schedule**, a versioned
|
||||
**schema**, a durable **checkpoint**, and a `status`.
|
||||
|
||||
`status` starts at **`pending`**. It was `active` before — both repository
|
||||
implementations wrote `input.status ?? "active"` — which meant an
|
||||
unclassified source went live the moment it was registered. `activateSource`
|
||||
is now the only path to `active`, and it refuses unless ALL of these are on
|
||||
record:
|
||||
|
||||
- an `owner_id` (distinct from `created_by`: who is accountable, not who typed it),
|
||||
- an **unexpired** authorization granted by an accountable actor
|
||||
(`svc_*`/`system`/`automation` are rejected — a source may not authorize itself),
|
||||
- rights with a `license_id` **and** at least one `allowed_purpose`,
|
||||
- provenance with a sourcing `method`.
|
||||
|
||||
Schedule, schema and checkpoint describe HOW a source runs, not WHETHER it
|
||||
may, so they are deliberately not part of the gate.
|
||||
|
||||
The default is enforced in two places on purpose: in the repositories, and as
|
||||
`ALTER TABLE opt_sources ALTER COLUMN status SET DEFAULT 'pending'`, so a raw
|
||||
`INSERT` from psql cannot produce an unclassified-but-active source either
|
||||
(proved in the realinfra suite).
|
||||
|
||||
Routes: `GET/POST /api/0pt/sources/{id}/activate` (GET returns the
|
||||
classification without changing anything), `GET/PUT …/authorization`,
|
||||
`GET/PUT …/schedule`.
|
||||
|
||||
## P061 — the connector contract
|
||||
|
||||
Nine operations on one interface: `validate` (pure, no IO, fail-closed),
|
||||
`connect` (idempotent), `discover`, `health` (reports, never throws), `read`
|
||||
(honours `limit` exactly), `checkpoint`, `resume`, `backfill` (bounded — an
|
||||
open-ended range is refused), `disconnect` (idempotent, safe after an error).
|
||||
|
||||
`src/connectors/conformance.ts` is a plain async function, not a test file,
|
||||
so the same suite runs from tests, from an operator script against a real
|
||||
configured source, and (later) from a pre-activation check. There is no
|
||||
"skip" verdict: a connector that cannot honestly serve a check FAILS it.
|
||||
|
||||
Two heterogeneous connectors pass it:
|
||||
|
||||
| | `HttpJsonConnector` | `JsonlFileConnector` |
|
||||
|---|---|---|
|
||||
| transport | network, real HTTP | local filesystem |
|
||||
| cursor | opaque server token, with a `cursor_field` fallback | line offset it computes itself |
|
||||
| exhaustion | server-reported | exact (the file is finite) |
|
||||
| health | HTTP round trip | `stat` |
|
||||
| backfill bounds | server-defined (sequence/timestamp) | line numbers |
|
||||
| CDC deletes | no — it does not claim `cdc` | yes, via `_op` |
|
||||
|
||||
The `cursor_field` fallback exists because the conformance suite caught a
|
||||
real defect: an API that stops returning `next_cursor` at the end of a page
|
||||
left the connector with nothing to resume from, so `resume()` silently
|
||||
restarted at the beginning.
|
||||
|
||||
## P062 — one envelope, one durable checkpoint
|
||||
|
||||
`IngestionEnvelope` is the single shape everything downstream sees:
|
||||
`mode` (batch/stream/cdc), `operation` (upsert/insert/update/delete),
|
||||
`partition_key`, `record_key`, `schema_version`, a monotonic `sequence`,
|
||||
`occurred_at`/`ingested_at`, `payload` (null only for a delete) and
|
||||
`provenance` (connector id + version + run id + cursor).
|
||||
|
||||
**Ordering is the contract.** Per page:
|
||||
|
||||
```
|
||||
read/resume → normalize → schema conformance → quality gate
|
||||
→ ledger PEEK → SINK → ledger COMMIT → checkpoint COMMIT
|
||||
```
|
||||
|
||||
Both commits happen after the sink resolves. The two tempting shortcuts are
|
||||
both data-loss bugs, and one of them was written here first — see STATUS.md's
|
||||
"A real bug this pass wrote and then caught".
|
||||
|
||||
The resulting guarantee is **at-least-once**, not exactly-once.
|
||||
`envelope_id` is a deterministic hash of
|
||||
`(source, partition, record_key, content)` so a dedup-capable sink survives a
|
||||
re-delivery.
|
||||
|
||||
## P063 — schema evolution
|
||||
|
||||
`RecordSchema` is a small structural shape (field → type + required) plus a
|
||||
`CompatibilityPolicy` (`backward` / `forward` / `full` / `none`).
|
||||
`checkCompatibility` answers what breaks, in which direction, and why. It is
|
||||
deliberately not a second JSON-Schema dialect: the question is "is version
|
||||
N+1 of an UPSTREAM shape safe for consumers of version N?", which needs a
|
||||
comparable structure, not a validator.
|
||||
|
||||
Two enforcement points, both upstream of any consumer write:
|
||||
|
||||
1. the runner's **discovery gate** — a shape change that breaks the previous
|
||||
version's declared policy fails the whole run before the first `read()`;
|
||||
2. **per-record conformance** — a source that drifted without redeclaring its
|
||||
schema is caught on the offending record, before the page reaches the sink.
|
||||
|
||||
A producer cannot widen its own policy: the PREVIOUS version's policy governs
|
||||
whether a change is allowed.
|
||||
|
||||
## P064 — quality with consequences
|
||||
|
||||
`QualityGate` composes the existing `ExpectationEngine` (the Great
|
||||
Expectations core subset already in this repo) and adds the missing half: a
|
||||
declared `on_violation` action.
|
||||
|
||||
- `block` — the page fails, and the checkpoint does **not** advance, so the
|
||||
bad page stays unconsumed rather than being stepped over.
|
||||
- `quarantine` — failing envelopes are persisted whole to
|
||||
`opt_quality_quarantine` (recoverable, not merely counted); clean ones
|
||||
continue and the checkpoint advances.
|
||||
- `warn` — everything continues, violations are counted and reported.
|
||||
|
||||
There is no default. With no policy the run report says
|
||||
`quality_policy: "none_declared"` — an undeclared policy must never read as a
|
||||
passed one. An expectation the local engine cannot execute counts as a
|
||||
VIOLATION, never a silent pass.
|
||||
|
||||
## P067 — queryable lineage
|
||||
|
||||
`LineageEmitter` keeps emitting OpenLineage to AUD0, and **AUD0 remains the
|
||||
system of record**. `LineageStore` adds 0pt's own flattened projection — one
|
||||
row per (run, input, output) with a deterministic `edge_id` — because
|
||||
"emitted to AUD0" was never the same thing as "answerable from 0pt".
|
||||
|
||||
```
|
||||
GET /api/0pt/lineage?tenant_id=…&output_name=0pt:dataset:ds_x # what fed it
|
||||
GET /api/0pt/lineage?tenant_id=…&input_name=source/src_x # what it produced
|
||||
```
|
||||
|
||||
An unfiltered dump is refused (400): that is an export, not a query.
|
||||
|
||||
## P068 — bounded, idempotent backfill
|
||||
|
||||
`opt_ingestion_ledger` holds one row per `(source_id, record_key)` with the
|
||||
last accepted content hash. A backfill runs through the same envelope,
|
||||
schema, quality and ledger path as a normal run, and:
|
||||
|
||||
- **never touches the live checkpoint** — a backfill must not rewind or
|
||||
advance streaming ingestion;
|
||||
- **re-emits nothing** on a repeat — every record comes back `duplicate`;
|
||||
- **still re-emits a genuine content change** — idempotency is not amnesia;
|
||||
- **refuses an unbounded range** at the connector contract level.
|
||||
|
||||
## Running the real-infrastructure proof
|
||||
|
||||
```bash
|
||||
docker run -d --name opt-ingestion-pgtest \
|
||||
-e POSTGRES_USER=opt -e POSTGRES_PASSWORD=opt_test_pw -e POSTGRES_DB=opt_test \
|
||||
-p 15434:5432 postgres:16-alpine
|
||||
|
||||
INGESTION_TEST_DATABASE_URL=postgres://opt:opt_test_pw@localhost:15434/opt_test \
|
||||
pnpm vitest run tests/p0/ingestion-plane-postgres.realinfra.test.ts
|
||||
```
|
||||
|
||||
The suite applies `sql/002_source_registry.sql` and
|
||||
`sql/007_ingestion_plane.sql` itself (both are idempotent). It skips
|
||||
honestly, never fabricating success, when the env var is unset.
|
||||
@@ -0,0 +1,41 @@
|
||||
# API deprecation policy — 0pt
|
||||
|
||||
BACKLOG P6 (API-24). There is exactly one prior breaking-ish change in this
|
||||
repo's history to learn from: `services/api/orchestrator/main.py` moved from
|
||||
`src/orchestrator` (docs/09-build-plan.md Phase 0) — internal, pre-contract,
|
||||
no external consumers yet. This policy is written before it's needed for a
|
||||
real consumer, on purpose (BACKLOG's own framing: "cheaper to write the
|
||||
policy than to clean up after breaking someone").
|
||||
|
||||
## What counts as breaking
|
||||
|
||||
- Removing a path, operation, or required request/response field from
|
||||
`contracts/openapi.yaml`.
|
||||
- Narrowing an accepted value (enum shrink, type tightening) on a request field.
|
||||
- Changing a response field's meaning without changing its name/type.
|
||||
- Changing default auth/tenant behavior on an existing route (e.g. what
|
||||
BACKLOG P0-4 fixed — the reverse of a break, but the same class of change).
|
||||
|
||||
Not breaking: adding an optional field, adding a new path, widening an
|
||||
accepted value, adding a new enum member a client can ignore.
|
||||
|
||||
## Process
|
||||
|
||||
1. Mark the field/path `deprecated: true` in `contracts/openapi.yaml` with an
|
||||
`x-siax-deprecated-since` date and a one-line replacement pointer.
|
||||
2. Keep the deprecated shape working for **at least one release cycle**
|
||||
after the replacement ships (see `CHANGELOG.md` once real releases/tags
|
||||
exist — BACKLOG P1-3).
|
||||
3. Announce in `CHANGELOG.md`'s `[Unreleased]` section before merge, not
|
||||
after.
|
||||
4. Only remove once nothing in this repo's own consumers (Protosell/FiveCRM,
|
||||
VehIQ per `CLAUDE.md`) is verified to depend on the old shape — "verified"
|
||||
means checked against those repos, not assumed silent.
|
||||
|
||||
## What this repo does NOT have yet
|
||||
|
||||
No consumers are known to be live against this contract yet
|
||||
(`docs/architecture/ARCHITECTURE.md` §2 "Beroenden nedåt": "Inga sådana
|
||||
anrop finns i det här repot" — unverified downstream). This policy exists
|
||||
so the first real breaking change has a process to follow, not because one
|
||||
has happened.
|
||||
@@ -0,0 +1,26 @@
|
||||
# API docs — 0pt
|
||||
|
||||
BACKLOG P1-20 (DOC-07/08). This directory did not exist before this pass.
|
||||
|
||||
**Source of truth:** [`contracts/openapi.yaml`](../../contracts/openapi.yaml)
|
||||
(OpenAPI 3.1, `0pt Data OS API`). Route handlers in `app/api/*` realize
|
||||
these contracts; `x-siax-status` on each path says whether it's `live` or
|
||||
contract-first (target).
|
||||
|
||||
This directory does **not** duplicate the contract as hand-written prose —
|
||||
that would drift the moment one of them is edited and not the other (exactly
|
||||
the kind of stale-doc problem this whole compliance pass exists to fix).
|
||||
Instead:
|
||||
|
||||
| Question | Where to look |
|
||||
|---|---|
|
||||
| What routes exist, request/response shapes | [`contracts/openapi.yaml`](../../contracts/openapi.yaml) directly, or render it with any OpenAPI viewer |
|
||||
| Which routes are live vs. target-only | `x-siax-status` field per path in the same file |
|
||||
| Risk level + auth requirement per route | `docs/architecture/ARCHITECTURE.md` §3 "API-yta" |
|
||||
| Capability → surface (API/CLI/MCP/UI) coverage | `docs/architecture/ARCHITECTURE.md` §5 "Ytor och paritet" |
|
||||
| The separate FastAPI engine contract (`services/api`) | `docs/04-api.md` |
|
||||
| The edge-native P0 capability contract (`app/api/0pt/*`) | `docs/12-edge-capability-api.md` |
|
||||
|
||||
There are genuinely **two** API surfaces documented separately (edge Next.js
|
||||
routes vs. the FastAPI engine) — see `docs/02-architecture.md` "Two API
|
||||
surfaces" for why, before assuming a route lives in the file you expected.
|
||||
@@ -1,11 +1,11 @@
|
||||
# 0PT — Arkitektur
|
||||
|
||||
**Kategori:** platform (K1) **Underkategori:** data-os **Status:** under utveckling (deployad frontend i drift, capability-lagret ej produktionsklart)
|
||||
**Senast verifierad mot kod:** 2026-08-07 (commit `6531384`, main) · **Klassificeringsuppdatering:** 2026-08-10 (`siax.repo.json`)
|
||||
**Senast verifierad mot kod:** 2026-08-07 (commit `6531384`, main) · **Klassificeringsuppdatering:** 2026-08-10 (`siax.repo.json`) · **v2.0-standardmigrering (§5 tillagd, §6–§9 omnumrerade; §6 Deploy-fältet nedan uppdaterat mot `lib/health-checks.ts`):** 2026-08-11 — scoped till dessa avsnitt, ingen fullständig radrad-omverifiering av hela dokumentet gjord denna pass
|
||||
|
||||
> **Läsanvisning.** Detta dokument beskriver vad koden i repot **gör**, inte vad
|
||||
> `README.md`, `CLAUDE.md` eller `docs/01–11` säger att den borde göra. Där de
|
||||
> avviker står avvikelsen utskriven — se §7. Påståenden märkta *(live-verifierat
|
||||
> avviker står avvikelsen utskriven — se §8. Påståenden märkta *(live-verifierat
|
||||
> 2026-08-07)* är kontrollerade mot den körande instansen på `https://0pt.app`.
|
||||
> Allt annat är läst ur källkoden. Där något inte gick att belägga står **OKÄNT**.
|
||||
|
||||
@@ -29,7 +29,7 @@ proveniens (källa, färskhet, konfidens) per fält.
|
||||
| consumes | CL0UD · ST0RE · N0D · AUD0 · INF0 |
|
||||
| replacesSaaS | Apollo · Cognism · ZoomInfo · Clearbit (målet för Data OS: ersätta paid enrichment-SaaS) |
|
||||
| decoupled | `false` |
|
||||
| designSystem | `null` (eget mono/editorial-system, konsumerar inte @siax/ui — öppen fråga §8.8) |
|
||||
| designSystem | `null` (eget mono/editorial-system, konsumerar inte @siax/ui — öppen fråga §9.8) |
|
||||
|
||||
Ersätter krav på betald enrichment-SaaS (Apollo/Cognism/ZoomInfo/Clearbit) med
|
||||
egen compliant sourcing + provenance — målet är att Datahub/VehIQ/data-avdelningar
|
||||
@@ -63,7 +63,7 @@ Tor-rotation, `cloudscraper`, `curl_cffi` TLS-fingerprintbyte, gratis proxypool,
|
||||
nodriver ("undetected browser") och "fake identity generation".
|
||||
`services/infra/docker-compose.data-stack.yml` startar `flaresolverr` och
|
||||
`tor-proxy` som tjänster. Commit `4a164c5` (Simon Axelsson, 2026-07-13) heter
|
||||
ordagrant *"refactor: remove compliance restrictions"*. Se §7 lucka **S9**.
|
||||
ordagrant *"refactor: remove compliance restrictions"*. Se §8 lucka **S9**.
|
||||
|
||||
---
|
||||
|
||||
@@ -74,8 +74,8 @@ ordagrant *"refactor: remove compliance restrictions"*. Se §7 lucka **S9**.
|
||||
| Beroende | Klient | Env-var | Beteende när env saknas |
|
||||
|---|---|---|---|
|
||||
| Zitadel (IdP) | `src/auth/zitadel.ts` | `ZITADEL_ISSUER` (default `https://id-customers.siax.io`) | Använder default-issuern; JWKS hämtas via OIDC-discovery, fallback `<issuer>/oauth/v2/keys` |
|
||||
| CL0UD (policy) | `src/clients/cl0ud-client.ts` | `CL0UD_URL` | **Fail-open** — returnerar alltid `allow` utan nätanrop (§7 S5) |
|
||||
| AUD0 (audit) | `src/clients/aud0-client.ts` | `AUD0_URL` | **No-op** — inga auditevent skrivs (§7 S6) |
|
||||
| CL0UD (policy) | `src/clients/cl0ud-client.ts` | `CL0UD_URL` | **Fail-open** — returnerar alltid `allow` utan nätanrop (§8 S5) |
|
||||
| AUD0 (audit) | `src/clients/aud0-client.ts` | `AUD0_URL` | **No-op** — inga auditevent skrivs (§8 S6) |
|
||||
| ST0RE (artefakter/lakehouse) | `src/clients/st0re-client.ts`, `st0re-lakehouse-client.ts` | `ST0RE_URL` | Returnerar deterministisk mockartefakt |
|
||||
| N0D (jobb) | `src/clients/n0d-client.ts` | `N0D_URL` | Returnerar deterministiskt mockjobb |
|
||||
| Egen data-engine | `lib/engine.ts` | `ENGINE_URL`, `INTERNAL_TOKEN` | `engineFetch` returnerar `null`, edge räknar lokalt |
|
||||
@@ -86,7 +86,7 @@ ordagrant *"refactor: remove compliance restrictions"*. Se §7 lucka **S9**.
|
||||
| Odoo | `src/adapters/odoo-bills-adapter.ts` | `ODOO_BASE_URL` | Adapter inaktiv |
|
||||
|
||||
Mönstret är genomgående: **saknad env ⇒ tyst degradering, inte fel.** Det är
|
||||
bekvämt lokalt och farligt i produktion (§7 S5, S6).
|
||||
bekvämt lokalt och farligt i produktion (§8 S5, S6).
|
||||
|
||||
### Beroenden nedåt (vem konsumerar 0pt)
|
||||
|
||||
@@ -98,7 +98,7 @@ här. Betrakta konsumentlistan som odokumenterad tills motparten visats.
|
||||
|
||||
- **BigQuery** — hård estate-regel. Ej överträdd.
|
||||
- **Hogia** — skrivning förbjuden. Ingen referens finns.
|
||||
- **Protosell-tenantdata** — tenantisolering, se §7 S2.
|
||||
- **Protosell-tenantdata** — tenantisolering, se §8 S2.
|
||||
- Nya SaaS-beroenden utan dokumenterat skäl (`CLAUDE.md` §4, "one tool per job").
|
||||
|
||||
---
|
||||
@@ -116,10 +116,10 @@ Risknivå = SIAX L0–L5-taxonomin, bedömd av mig utifrån vad handlern gör.
|
||||
| Registrera dataset↔Iceberg-mappning | `POST /api/0pt/lakehouse/mappings` | L3 | Nej i kod; `tenant_id` läses ur body |
|
||||
| Lista mappningar | `GET /api/0pt/lakehouse/mappings` | L2 | Nej; `tenant_id` ur query |
|
||||
| Hämta mappning | `GET /api/0pt/lakehouse/mappings/{mappingId}` | L2 | Nej |
|
||||
| Utför Iceberg-write | `POST /api/0pt/lakehouse/mappings/{mappingId}/writes` | **L4** | Nej i kod (CL0UD fail-open, §7 S5) |
|
||||
| Utför Iceberg-write | `POST /api/0pt/lakehouse/mappings/{mappingId}/writes` | **L4** | Nej i kod (CL0UD fail-open, §8 S5) |
|
||||
| Registrera rate-policy | `POST /api/0pt/sources/rate-policies` | L2 | Nej |
|
||||
| Lista/hämta rate-policy | `GET /api/0pt/sources/rate-policies[/{id}]` | L1 | Nej |
|
||||
| Kör källfråga | `POST /api/0pt/sources/{sourceId}/query` | **L4** | Nej; `connection_config` tas ur body (§7 S10) |
|
||||
| Kör källfråga | `POST /api/0pt/sources/{sourceId}/query` | **L4** | Nej; `connection_config` tas ur body (§8 S10) |
|
||||
| Ta emot dokument | `POST /api/0pt/documents/capture` | L3 (PII) | Nej |
|
||||
| Lista dokument | `GET /api/0pt/documents/capture` | L3 (PII) | Nej |
|
||||
| Extrahera dokument | `POST /api/0pt/documents/{documentId}/extraction` | L3 (PII) | Nej |
|
||||
@@ -132,6 +132,7 @@ Risknivå = SIAX L0–L5-taxonomin, bedömd av mig utifrån vad handlern gör.
|
||||
| Operation | Metod/sökväg | Risknivå | Kräver godkännande |
|
||||
|---|---|---|---|
|
||||
| Liveness | `GET /api/health` | L0 | Nej (avsiktligt) |
|
||||
| Readiness | `GET /api/ready` | L0 | Nej (avsiktligt) |
|
||||
| Prisestimat | `POST /api/estimates` | L1 | Nej |
|
||||
| Katalog / källor / fordon | `GET /api/{catalog,sources,vehicles}` | L1 | Nej |
|
||||
| Sanktions-/PEP-screening | `GET /api/screen?q=` | L2 | Nej — proxar mot OpenSanctions |
|
||||
@@ -141,11 +142,11 @@ Risknivå = SIAX L0–L5-taxonomin, bedömd av mig utifrån vad handlern gör.
|
||||
| Läs/ändra order | `GET,PATCH /api/orders/{id}` | L3 | Nej |
|
||||
| **Adminmetrik (orderpipeline, belopp)** | `GET /api/admin/metrics` | **L3** | **Nej** *(live-verifierat: 200 anonymt)* |
|
||||
| Deposition (Stripe checkout) | `POST /api/payments/{order_id}/deposit` | **L4** | Nej |
|
||||
| **Frigör escrow-slutbetalning** | `POST /api/payments/{order_id}/release` | **L4** | **Nej** — sätter dessutom status `delivered` (§7 S4) |
|
||||
| **Frigör escrow-slutbetalning** | `POST /api/payments/{order_id}/release` | **L4** | **Nej** — sätter dessutom status `delivered` (§8 S4) |
|
||||
| Stripe-webhook | `POST /api/webhooks/stripe` | L3 | Signaturkontroll via `STRIPE_WEBHOOK_SECRET` |
|
||||
| Prenumerationer | `GET,POST /api/subscriptions` | L2 | Nej |
|
||||
| Filuppladdning | `GET,POST /api/uploads` | L3 | Nej (`501` utan R2) |
|
||||
| Ta emot kund-DB-grant | `POST /api/connections/db-grant` | **L4** | Nej (`501` utan `GRANT_ENC_KEY`; §7 S7) |
|
||||
| Ta emot kund-DB-grant | `POST /api/connections/db-grant` | **L4** | Nej (`501` utan `GRANT_ENC_KEY`; §8 S7) |
|
||||
| Dev-login | `POST /api/auth/login` | L2 | Ja — kräver `AUTH_SECRET` + matchande `DEV_LOGIN_KEY` |
|
||||
| Logout / me | `POST /api/auth/logout`, `GET /api/auth/me` | L1 | Cookiebaserat |
|
||||
|
||||
@@ -156,7 +157,7 @@ Risknivå = SIAX L0–L5-taxonomin, bedömd av mig utifrån vad handlern gör.
|
||||
`PATCH /orders/{oid}`, `POST /orders/{oid}/run`, `GET /orders/{oid}/status`.
|
||||
**Ingen autentisering alls** — grep efter `Depends`, `auth`, `token`,
|
||||
`X-Internal-Token` i filen ger noll träffar, trots att `lib/engine.ts` skickar
|
||||
`X-Internal-Token`. Se §7 S8.
|
||||
`X-Internal-Token`. Se §8 S8.
|
||||
|
||||
---
|
||||
|
||||
@@ -208,7 +209,7 @@ Två konsekvenser, båda dokumenterade i koden själv:
|
||||
|
||||
- `dim_contact`: namn, roll, e-post, telefon, LinkedIn — persondata under GDPR.
|
||||
- `dim_ubo`: verklig huvudman — persondata, potentiellt känslig.
|
||||
- `db_grants.conn_encrypted`: kunds DB-anslutningssträng (se §7 S7).
|
||||
- `db_grants.conn_encrypted`: kunds DB-anslutningssträng (se §8 S7).
|
||||
- Dokumentcapture (`app/api/0pt/documents/capture`) tar emot godtyckliga
|
||||
dokument, inkl. fakturor/mailbilagor via Graph-adaptern — persondata.
|
||||
|
||||
@@ -220,7 +221,50 @@ inget sådant jobb, ingen TTL, ingen `DELETE`-väg och ingen radering finns i ko
|
||||
|
||||
---
|
||||
|
||||
## 5. Deploy
|
||||
## 5. Ytor och paritet
|
||||
|
||||
**Migrering till standard v2.0 (BACKLOG P1-20/DOC-02):** denna sektion är ny —
|
||||
v1.0-mallen hade 8 rubriker, v2.0 har 9 (Deploy/Drift/Säkerhet/Öppna frågor
|
||||
numrerade om §6–§9 nedan, tidigare §5–§8).
|
||||
|
||||
Källa: `cl0ud packages/capability-registry/src/registry.ts` (samma verifiering
|
||||
som `docs/02-architecture.md`s "Six-surface capability model"-avsnitt, ej
|
||||
omgjord här). 14 `0pt.*`-kapabiliteter är registrerade. Registret har fält för
|
||||
`api`/`cli`/`mcp`/`ui`/`job`; "CL0UD-konsol" nedan är en separat fråga (om
|
||||
kapabiliteten är inbäddad i den delade cl0ud-konsolen) som INTE går att
|
||||
verifiera från detta repo — markerad OKÄNT genomgående, inte gissad.
|
||||
|
||||
"Egen konsol" är **ANTAGANDE baserat på sidnamn/routestruktur under `app/`**,
|
||||
inte spårat anrop-för-anrop mot varje handler — verifiera innan den läses som
|
||||
hårt bevisad.
|
||||
|
||||
| Kapabilitet | API | CLI | MCP | Egen konsol | CL0UD-konsol |
|
||||
|---|---|---|---|---|---|
|
||||
| `0pt.order_create` | ✅ | ❌ | ❌ | ✅ (`app/bestall/*`) | OKÄNT |
|
||||
| `0pt.order_payment` | ✅ | ❌ | ❌ | ✅ (`app/bestall/forskott`) | OKÄNT |
|
||||
| `0pt.estimate_compute` | ✅ | ❌ | ❌ | ✅ (`app/bestall/beskriv`, ANTAGANDE) | OKÄNT |
|
||||
| `0pt.catalog_list` | ✅ | ❌ | ❌ | ✅ (`app/katalog`) | OKÄNT |
|
||||
| `0pt.source_registry_list` | ✅ | ❌ | ❌ | ✅ (`app/kallor`, `app/admin/sources`) | OKÄNT |
|
||||
| `0pt.query_source` | ✅ | ❌ | ❌ | ❌ (ingen dedikerad sida funnen) | OKÄNT |
|
||||
| `0pt.vehicle_lookup` | ✅ | ❌ | ❌ | ❌ (ingen dedikerad sida funnen) | OKÄNT |
|
||||
| `0pt.screen_sanctions` | ✅ | ❌ | ❌ | ❌ (ingen dedikerad sida funnen) | OKÄNT |
|
||||
| `0pt.document_ocr` | ✅ | ❌ | ❌ | ❌ (ingen dedikerad sida funnen) | OKÄNT |
|
||||
| `0pt.lakehouse_mapping_manage` | ✅ | ❌ | ❌ | ❌ (ingen dedikerad sida funnen) | OKÄNT |
|
||||
| `0pt.source_rate_policy_manage` | ✅ | ❌ | ❌ | ❌ (ingen dedikerad sida funnen) | OKÄNT |
|
||||
| `0pt.lakehouse_write` | ✅ (+job) | ❌ | ❌ | ❌ (ingen dedikerad sida funnen) | OKÄNT |
|
||||
| `0pt.write_iceberg_table` | ✅ (+job) | ❌ | ❌ | ❌ (ingen dedikerad sida funnen) | OKÄNT |
|
||||
| `0pt.write_external_record` | ❌ (job-only, `apiEnabled: false`) | ❌ | ❌ | ❌ | OKÄNT |
|
||||
|
||||
**Luckor, sammanfattat:** CLI = 0/14, MCP = 0/14 (matchar BACKLOG P1-5/P1-6:
|
||||
ingen CLI-yta, ingen MCP-yta alls finns i detta repo ännu). API = 13/14 (allt
|
||||
utom det renodlade job-kapabiliteten). Egen konsol ≈ 5/14, koncentrerat till
|
||||
order-/katalog-/källflödet; kapabiliteter under `app/api/0pt/*` (lakehouse,
|
||||
rate-policy, källfråga, sanktionsscreening, dokument-OCR) har ingen synlig UI
|
||||
alls — matchar BACKLOG P1-8 ("`/api/0pt/*`-kapabiliteterna saknar UI").
|
||||
|
||||
---
|
||||
|
||||
## 6. Deploy
|
||||
|
||||
| Fält | Verifierat läge |
|
||||
|---|---|
|
||||
@@ -233,18 +277,27 @@ inget sådant jobb, ingen TTL, ingen `DELETE`-väg och ingen radering finns i ko
|
||||
| **CI** | `.gitea/workflows/ci.yml`. `quality`-jobb: typecheck (blockerande, endast `src/` + `tests/`), vitest (blockerande, 17 testfiler), eslint (rådgivande), `next build` (blockerande). `security`-jobb: Trivy `--exit-code 0` (rådgivande). Kör på `ubuntu-latest`. |
|
||||
|
||||
> **Konflikt att fatta beslut om:** estate-projektet *GCP/Vercel-exit* säger att
|
||||
> flottan ska bort från Vercel. 0pt:s enda körande instans ligger på Vercel. Se §8.
|
||||
> flottan ska bort från Vercel. 0pt:s enda körande instans ligger på Vercel. Se §9.
|
||||
|
||||
---
|
||||
|
||||
## 6. Drift
|
||||
## 7. Drift
|
||||
|
||||
### Hälsokontroll
|
||||
`GET /api/health` → `{ok, service:"0pt-web", db, queue, ts}`. Den svarar **alltid
|
||||
`ok:true`** oavsett om databasen finns — den rapporterar bara `"not_configured"`
|
||||
i ett fält. En uptime-monitor som tittar på HTTP 200 kommer därför visa grönt
|
||||
för en instans utan databas, vilket är exakt vad som gäller idag. Engine har ett
|
||||
separat `GET /health`.
|
||||
**Fixat (BACKLOG P0-3):** `GET /api/health` → `{ok, service:"0pt-web", db, queue,
|
||||
engine, stripe, ts}`. `db`/`engine`/`stripe` rapporterar nu en riktig
|
||||
beroendeprobe (`lib/health-checks.ts`): `"ok"` = lyckad live-anrop, `"error"` =
|
||||
konfigurerad men onåbar (se `detail`), `"not_configured"` = env-variabel ej
|
||||
satt. `queue` kan bara vara `"not_configured"`/`"not_integrated"` — ingen
|
||||
Redis-klient finns i repot, så en satt `REDIS_URL` rapporteras ärligt i stället
|
||||
för att låtsas testa en anslutning som aldrig görs. `ok` är `true` endast när
|
||||
ingen konfigurerad dependency ligger i `"error"`. HTTP-statusen är fortsatt
|
||||
alltid 200 (liveness — processen svarar) oavsett dependency-läge; en extern
|
||||
beroendekrasch ska inte omstarta instansen. Ny `GET /api/ready` failar stängt
|
||||
(503) när databasen (den enda lastbärande dependencyn idag) inte både är
|
||||
konfigurerad och nåbar — peka en Coolify/lastbalanserar-readiness-probe dit,
|
||||
inte mot `/api/health`. Engine har ett separat `GET /health` som `checkEngine()`
|
||||
nu faktiskt anropar (tidigare kollades bara `ENGINE_URL`-närvaro).
|
||||
|
||||
### Loggar
|
||||
Ingen strukturerad loggning. `console.warn` i `src/auth/context.ts` är den enda
|
||||
@@ -277,7 +330,7 @@ Om/när Postgres provisioneras måste detta lösas separat — jfr estate-blocke
|
||||
|
||||
---
|
||||
|
||||
## 7. Säkerhet
|
||||
## 8. Säkerhet
|
||||
|
||||
### Vad som faktiskt finns
|
||||
|
||||
@@ -396,7 +449,7 @@ bindning till en användarpost. Kommentaren erkänner att det är ett scaffold;
|
||||
|
||||
---
|
||||
|
||||
## 8. Öppna frågor
|
||||
## 9. Öppna frågor
|
||||
|
||||
1. **Varför kör inte middlewaren i produktion (S1)?** Måste rotorsaksanalyseras.
|
||||
Ingen dokumentation får påstå att `/api/0pt/*` är skyddat innan en anonym
|
||||
@@ -408,7 +461,7 @@ bindning till en användarpost. Kommentaren erkänner att det är ett scaffold;
|
||||
3. **Vercel eller flottan?** *(config-delen löst 2026-08-09, trafik-delen kvarstår.)*
|
||||
0pt:s enda **körande** instans ligger fortfarande på Vercel — den ändringen är
|
||||
medvetet INTE gjord här (skulle kräva en egen cutover-plan: verifiera att inget
|
||||
riktigt kunddata/trafik ligger bakom `0pt.app` — vilket är fallet idag, se §5/§6:
|
||||
riktigt kunddata/trafik ligger bakom `0pt.app` — vilket är fallet idag, se §6/§7:
|
||||
`/api/health` visar `db/queue: not_configured`, ordrar lever bara in-memory —
|
||||
provisionera en Coolify-app, peka DNS om, och avveckla Vercel-projektet, i den
|
||||
ordningen). Det som ÄR gjort: `siax.config.json` deklarerar nu `target: "coolify"`
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
# ADR-0001: Deploy target is Coolify (SIAX Cloud fleet), not Vercel
|
||||
|
||||
- **Status:** Accepted (config declares it; migration not yet executed)
|
||||
- **Date decided:** 2026-08-09 (commit `ad042d8`, "fix(deploy): declare Coolify as the deploy target, not Vercel")
|
||||
- **Date of this record:** 2026-08-11 (BACKLOG P1-2 — this ADR documents an
|
||||
already-made decision retroactively; it does not introduce a new one)
|
||||
|
||||
## Context
|
||||
|
||||
0pt was originally scaffolded and deployed on Vercel (Hobby/free tier,
|
||||
project `0pt`, scope *Simon Axelsson's projects*). SIAX Cloud's platform
|
||||
mandate is to replace paid SaaS with self-hosted fleet infrastructure
|
||||
(Hetzner + Coolify), matching every other SIAX Cloud OS repo (`n0d`,
|
||||
`st0re`, `aud0`, ...).
|
||||
|
||||
## Decision
|
||||
|
||||
`siax.config.json`'s `deploy.target` is `"coolify"`, not `"vercel"`. A
|
||||
Dockerfile written for Coolify (multi-stage, non-root `node` user, `HEALTHCHECK`
|
||||
against `/api/health`) and `.gitea/workflows/cd.yml` (POSTs a restart to a
|
||||
Coolify app UUID on push to `main`) both exist and are committed.
|
||||
|
||||
## Consequences
|
||||
|
||||
- **This is a config-and-tooling change, not a traffic cutover.** As of this
|
||||
record, no Coolify application has been provisioned for this repo —
|
||||
`cd.yml`'s `COOLIFY_APP_ID` is a literal placeholder that fails the job
|
||||
loudly on purpose (see the guard step) rather than posting to a bogus
|
||||
endpoint. The live instance of `0pt.app` is still served by Vercel.
|
||||
|
||||
> **UPDATE 2026-09-06 (the consequence above has since been overtaken; the
|
||||
> decision itself stands).** **CORRECTED 2026-09-06 — this is no longer true.** The Coolify deploy hook is LIVE: the repo secret `COOLIFY_TOKEN` exists (created 2026-08-31T11:34:34Z) and the repo Actions variable `COOLIFY_APP_ID` holds a real 24-character UUID, not the `TODO-no-coolify-app-provisioned-yet` literal. Both guard conditions in `.gitea/workflows/cd.yml` therefore pass, and the push to `main` at `1ff34410` (2026-09-06) ran `Deploy to Coolify / deploy` to **success** with `coolify deploy HTTP 200` in the job log. **Every merge into `main` now fires a real redeploy against cloud.siax.io.** What is still NOT verified: whether that Coolify app actually builds and serves, and whether `0pt.app` DNS points at it or still at Vercel — nobody has probed the running site.
|
||||
>
|
||||
> An ADR is a historical record, so the original consequence text is left
|
||||
> intact above rather than rewritten — read it as "true when written
|
||||
> 2026-08-09", not as current state.
|
||||
- Flipping the config was deliberately decoupled from the actual cutover: it
|
||||
states intent (BACKLOG P0-2 / P3-1 own the "provision the Coolify app, cut
|
||||
DNS, decommission Vercel" work, gated on a dump-before-write blocker per
|
||||
the standard's "safe to run now" classification).
|
||||
- Until the cutover happens, docs describing Vercel as *where 0pt actually
|
||||
runs today* are accurate; docs prescribing Vercel as *the plan going
|
||||
forward* are stale (see BACKLOG P2-5 — `docs/02-architecture.md`'s
|
||||
"Hosting" section is marked SUPERSEDED for exactly this reason).
|
||||
|
||||
## See also
|
||||
|
||||
- `docs/architecture/ARCHITECTURE.md` §6 "Deploy", §9 "Öppna frågor"
|
||||
- `README.md` "Deploy" section (kept current with the live/target split)
|
||||
- `docs/compliance/BACKLOG.md` P0-2, P1-17, P3-1
|
||||
@@ -0,0 +1,57 @@
|
||||
# ADR-0002: `/api/0pt/*` authenticates via fleet Zitadel Bearer tokens, not NextAuth/Auth.js
|
||||
|
||||
- **Status:** Accepted and implemented
|
||||
- **Date decided:** ~2026-07 ("Wave 2" — commit `1d76d50`,
|
||||
"feat(auth,foundation): Wave 2 flag-gated Zitadel JWT verify + validation
|
||||
+ eslint + tests"), fail-closed default landed commit `1c1e2cb`
|
||||
("fix(auth): make the AUTH_ENFORCE default fail-closed on `/api/0pt/*`")
|
||||
- **Date of this record:** 2026-08-11 (BACKLOG P1-2 — retroactive)
|
||||
|
||||
## Context
|
||||
|
||||
0pt's capability surface (`/api/0pt/*`) needs to authenticate requests
|
||||
against a real identity, and needs that identity to carry a SIAX tenant so
|
||||
`x-0pt-verified-tenant` can be trusted downstream. SIAX Cloud already runs a
|
||||
fleet-wide Zitadel IdP (`id-customers.siax.io`) for exactly this purpose —
|
||||
building a second, repo-local identity provider would duplicate it and
|
||||
create a second source of truth for "who is this".
|
||||
|
||||
## Decision
|
||||
|
||||
`middleware.ts` verifies inbound `Authorization: Bearer` tokens against the
|
||||
fleet Zitadel IdP (`src/auth/zitadel.ts`): JWKS discovered from
|
||||
`<issuer>/.well-known/openid-configuration` (falling back to
|
||||
`<issuer>/oauth/v2/keys`), tenant derived from a documented claim-key
|
||||
precedence (`urn:zitadel:iam:org:id` → `urn:zitadel:iam:user:resourceowner:id`
|
||||
→ `tenant_id` → `tenant`). No NextAuth/Auth.js dependency was added.
|
||||
|
||||
Enforcement is fail-closed by default: `AUTH_ENFORCE` unset (or any value
|
||||
except the literal string `"false"`) requires a valid, verified token — 401
|
||||
on missing/invalid, 403 when a client-supplied `X-Tenant-Id` disagrees with
|
||||
the token's own tenant claim. `AUTH_ENFORCE=false` is a local-dev-only escape
|
||||
hatch, and is refused at boot when `NODE_ENV=production`
|
||||
(`assertAuthConfigSafe()` throws rather than serving anonymously).
|
||||
|
||||
A separate, older HMAC session cookie (`lib/auth.ts`, with a `DEV_LOGIN_KEY`
|
||||
dev-login shortcut) exists in parallel and is explicitly **not** the auth
|
||||
system this ADR is about — it predates the Zitadel work, has no MFA/CSRF/
|
||||
revocation, and is tracked as its own gap (BACKLOG P0-14, gated on the
|
||||
account owner's decision about what replaces it).
|
||||
|
||||
## Consequences
|
||||
|
||||
- 0pt does not maintain its own user/credential store for the capability
|
||||
layer — identity is entirely delegated to the fleet Zitadel instance.
|
||||
- A live-verified gap was found and is tracked separately: a 2026-08-10
|
||||
probe found `GET 0pt.app/api/0pt` returning 200 anonymously despite the
|
||||
code being fail-closed (BACKLOG P0-4/P0-5, root cause unresolved as of
|
||||
this record — do not assume the deployed instance enforces this until an
|
||||
anonymous request gets a real 401 there).
|
||||
- `middleware.ts` sits outside `tsconfig.check.json`'s blocking type-check
|
||||
scope (BACKLOG P0-5) — a real gap, not addressed by this ADR.
|
||||
|
||||
## See also
|
||||
|
||||
- `docs/architecture/ARCHITECTURE.md` §8 "Säkerhet"
|
||||
- `tests/auth/{context,zitadel}.test.ts`
|
||||
- `docs/compliance/BACKLOG.md` P0-4, P0-5, P0-14
|
||||
@@ -0,0 +1,47 @@
|
||||
# ADR-0003: Lakehouse table format is Apache Iceberg (pointer to `n0d` ADR-0030)
|
||||
|
||||
- **Status:** Accepted (decision owned by `n0d`, not by this repo)
|
||||
- **Date decided:** documented in the `n0d` repo as ADR-0030; referenced in
|
||||
this repo since commit `2545218` ("Document lakehouse write-path ownership
|
||||
decision (ADR-0030)")
|
||||
- **Date of this record:** 2026-08-11 (BACKLOG P1-2 — this is a **pointer**
|
||||
ADR, not a duplicate decision; it exists so `docs/architecture/decisions/`
|
||||
is a complete index of decisions that shape this repo, even the ones made
|
||||
elsewhere)
|
||||
|
||||
## Context
|
||||
|
||||
BACKLOG P1-2 asks for ADRs to exist in this repo. The lakehouse-format
|
||||
decision itself was made once, ecosystem-wide, in `n0d` (the repo that owns
|
||||
the platform's write path across all K1 data producers) — re-deciding it
|
||||
here would create a second, divergent source of truth for a cross-repo
|
||||
architectural choice.
|
||||
|
||||
## Decision
|
||||
|
||||
Apache Iceberg is the canonical SIAX Cloud lakehouse table format. 0pt is a
|
||||
**write-path owner** within that decision: it maps its own datasets to
|
||||
Iceberg tables and requests writes through the documented boundary (N0D
|
||||
executes, 0pt requests — see `docs/ICEBERG_WRITE_BOUNDARY.md`). This repo
|
||||
does not re-litigate the Iceberg-vs-alternatives choice; it implements its
|
||||
own slice of it.
|
||||
|
||||
## Consequences
|
||||
|
||||
- `docs/10-lakehouse-decision.md` and `docs/11-lakehouse-write-plan.md` are
|
||||
this repo's own follow-up documentation against the `n0d`-owned decision —
|
||||
read them alongside this pointer, not instead of it.
|
||||
- `docs/ICEBERG_WRITE_BOUNDARY.md` records the implementation status,
|
||||
including which parts are live-verified vs. contract-only. That document,
|
||||
not this ADR, is the source of truth for "is the write path actually
|
||||
working" — this ADR only records *which format was chosen and by whom*.
|
||||
- Any future change to the ecosystem-wide format decision would originate in
|
||||
`n0d`, not here; this repo would need its own follow-up ADR to record how
|
||||
it adapts.
|
||||
|
||||
## See also
|
||||
|
||||
- `docs/10-lakehouse-decision.md`
|
||||
- `docs/11-lakehouse-write-plan.md`
|
||||
- `docs/ICEBERG_WRITE_BOUNDARY.md`
|
||||
- `n0d` repo, ADR-0030 (external — not vendored into this repo)
|
||||
@@ -0,0 +1,34 @@
|
||||
# Architecture Decision Records — 0pt
|
||||
|
||||
BACKLOG P1-2. One file per decision, numbered sequentially, never renumbered
|
||||
or deleted (superseded decisions get a new ADR that says so, per the
|
||||
convention below).
|
||||
|
||||
| ADR | Title | Status |
|
||||
|---|---|---|
|
||||
| [0001](0001-deploy-target-coolify-not-vercel.md) | Deploy target is Coolify, not Vercel | Accepted (migration not yet executed) |
|
||||
| [0002](0002-auth-zitadel-bearer-not-nextauth.md) | `/api/0pt/*` auth is fleet Zitadel Bearer, not NextAuth | Accepted and implemented |
|
||||
| [0003](0003-lakehouse-format-iceberg-pointer.md) | Lakehouse format is Apache Iceberg (pointer to `n0d` ADR-0030) | Accepted (owned by `n0d`) |
|
||||
|
||||
## Deliberately not yet written here
|
||||
|
||||
Two decisions the original BACKLOG (P1-2) named explicitly are **not**
|
||||
written as ADRs in this pass, on purpose:
|
||||
|
||||
- **Payments provider (Stripe vs. Revolut vs. Hyperswitch).** Gated on the
|
||||
account owner's decision (BACKLOG P1-23) — writing an ADR that presents a
|
||||
choice as "accepted" before it's actually been decided would misrepresent
|
||||
the repo's own state, which is exactly what this whole documentation pass
|
||||
is trying to stop doing.
|
||||
- **The `services/scrapers/*` keep-vs-remove question.** Explicitly out of
|
||||
scope for this pass — see `SECURITY.md` and the dated, attributed
|
||||
provenance record already maintained for that decision elsewhere in this
|
||||
repo's compliance docs. Not duplicated here.
|
||||
|
||||
## Convention
|
||||
|
||||
New ADR: `NNNN-short-kebab-title.md`, next sequential number, status one of
|
||||
`Proposed` / `Accepted` / `Superseded by ADR-NNNN` / `Rejected`. Record the
|
||||
date the decision was actually made (from `git log`/commit history where
|
||||
possible), not just the date the ADR file was written, when they differ —
|
||||
see 0001–0003 for the pattern.
|
||||
@@ -0,0 +1,72 @@
|
||||
# BLOCKERS.md — 0pt (§1.4)
|
||||
|
||||
> Policy: enligt SIAX-F0KUSREPO-PROMPT §1.4 (Blockerarpolicy) dokumenteras varje
|
||||
> blockerare som inte kan lösas själv eller plattformsbrett här — exakt vad som
|
||||
> krävs för att lyfta den. En obesvarad blockerare som inte står i denna fil
|
||||
> räknas som ett fel i leveransen (Appendix C #20).
|
||||
>
|
||||
> Märkning: 🔴 = öppen blockerare · 🟡 = beslutad/accepterad skuld (frågan är
|
||||
> avgjord, men konsekvensen kvarstår i koden/driften) · Bevis per post (AR-5).
|
||||
>
|
||||
> Verifierad färsk **2026-08-11** mot `main` @ `5c36dc300f45fed6ae789a5edccc0dfcbfb98918`
|
||||
> (`git.siax.io/sax3l/0pt`). Detta dokument fanns inte tidigare i repot
|
||||
> (bekräftad frånvaro vid en tidigare audit-passering) — denna fil är den
|
||||
> första skrivningen, inte en uppdatering.
|
||||
|
||||
---
|
||||
|
||||
## B-1 🟡 — WAF-bypass-/evasionstacken (`services/scrapers/*`)
|
||||
|
||||
| Fält | Innehåll |
|
||||
|---|---|
|
||||
| **Vad blockerar** | `services/scrapers/waf_bypass.py`, `engine.py` och `engine_v2.py` innehåller en FlareSolverr-/Tor-/proxyrotationsstack som strider mot `CLAUDE.md` §1:s uttryckliga förbud mot WAF/bot-detektionsbypass. En kirurgiskt ren borttagnings-PR (**#7**, `chore(compliance): ta bort den förbjudna WAF-bypass-/evasionstacken`) verifierades ren men stängdes **omergad** 2026-08-09. |
|
||||
| **Vad krävs för att lyfta den** | Ett uttryckligt, dokumenterat kontoägarbeslut: behåll, bygg om, eller ta bort stacken. |
|
||||
| **Vem beslutar** | Simon Axelsson (kontoägare). |
|
||||
| **Nuvarande status** | **Redan beslutad (behåll) — 2026-08-11.** Kontoägaren tillfrågades direkt via en interaktiv beslutsprompt, som svar på PR #7:s borttagningsförslag, och valde uttryckligen **behåll**. Beslutet är nu dokumenterat i `AGENTS.md` (rad ~21–30) och `SECURITY.md`, mergat till main via PR **#15** (docs-only, merge-commit `5c36dc3` — samma commit som är denna verifierings `main`-HEAD). PR #7 kvarstår **closed/unmerged** och rörs inte av detta dokument eller detta uppdrag. Ingen affärs- eller juridisk motivering utöver "behåll" är dokumenterad, och ingen ska hittas på om den saknas i framtida rapporter. `services/scrapers/{waf_bypass,engine,engine_v2}.py` är oförändrade av denna PR och ligger utanför detta dokuments och detta uppdrags mandat. |
|
||||
| **Bevis** | `git log -- services/scrapers/waf_bypass.py` (`bb6be0f`, `afd301f`); Gitea API `repos/sax3l/0pt/pulls/7` (`state: closed`, `merged: false`, `closed_at: 2026-08-09T09:49:05Z`); Gitea API `repos/sax3l/0pt/pulls/15` (`merged: true`, `merged_at: 2026-08-11T09:36:17Z`); `AGENTS.md` rad 21–30 (denna checkout). |
|
||||
|
||||
---
|
||||
|
||||
## B-2 🔴 — Ingen Coolify-app provisionerad, inget Infisical-projekt
|
||||
|
||||
| Fält | Innehåll |
|
||||
|---|---|
|
||||
| **Vad blockerar** | `.gitea/workflows/cd.yml` har fortfarande en literal platshållare: `COOLIFY_APP_ID: "TODO-no-coolify-app-provisioned-yet"`. Jobbet är medvetet guardat att lyckas grönt utan att faktiskt deploya (`exit 0` + varning om token/UUID saknas). `siax.repo.json`/`siax.config.json` saknar Infisical-referens. Detta matchar repots egna `docs/compliance/BACKLOG.md` (P1-5, P1-6) och `GOLDEN-STANDARD-REPORT.md` (D4 ❌, C8 ❌, D7 ➖). |
|
||||
| **Vad krävs för att lyfta den** | Skapa appen i `cloud.siax.io` (Coolify), sätt `COOLIFY_APP_ID` i `cd.yml` och `COOLIFY_TOKEN` som Gitea-repo-secret. Skapa ett Infisical-projekt `0pt` med env-mappning och migrera bort från Vercel-env i takt med cutover. |
|
||||
| **Vem beslutar** | Ingen produktbeslutsfråga i sig — operativt arbete som kräver admin-access till kontrollplanet (`cloud.siax.io` / `vault.siax.io`). |
|
||||
| **Nuvarande status** | Fortfarande öppen. Verifierad färsk denna session: `cd.yml` bär fortfarande TODO-platshållaren på nuvarande `main`-HEAD. Kontrollerat om ett parallellt P0-1-uppdrag i samma arbetsflöde redan hunnit provisionera en app/projekt — inget sådant syns i repots egna deploy-artefakter vid tidpunkten för denna verifiering. Denna session saknar `SIAX_TOKEN`/inloggning mot `cloud.siax.io`/Infisical, så ett levande skick bortom repots egna signaler kunde inte kontrolleras direkt — endast repo-interna bevis nedan är verifierade. |
|
||||
| **Bevis** | `.gitea/workflows/cd.yml` rad 25 (denna checkout); `docs/compliance/BACKLOG.md` P1-5/P1-6; `docs/compliance/GOLDEN-STANDARD-REPORT.md` D4/D7/C8. |
|
||||
|
||||
---
|
||||
|
||||
## B-3 🔴 — DNS-zonerna (`0pt.app`, `0pt.dev`) delegerade till Vercel, inte egen PowerDNS
|
||||
|
||||
| Fält | Innehåll |
|
||||
|---|---|
|
||||
| **Vad blockerar** | Live NS-uppslag (denna session, 2026-08-11) visar att både `0pt.app` och `0pt.dev` fortfarande delegeras till `ns1.vercel-dns.com`/`ns2.vercel-dns.com` — trots att PowerDNS (server5/server6, se z0ne-DNS-migrationen) redan håller skuggkopior av dessa zoner. |
|
||||
| **Vad krävs för att lyfta den** | Byt NS-poster hos registrar (Name.com) till `ns1.siax.io`/`ns2.siax.io`; verifiera att zoninnehållet i PowerDNS matchar produktionens faktiska poster innan cutover (dump/backup krävs per §1.4:s regel om irreversibla DNS-delegeringsbyten); ta därefter bort eller markera skuggkopiorna som primära. |
|
||||
| **Vem beslutar** | Simon Axelsson — DNS-cutover för en produktionsdomän räknas som en operation som kräver dump/backup före körning enligt §1.4. |
|
||||
| **Nuvarande status** | Fortfarande öppen. Verifierad färsk denna session (`nslookup -type=NS 0pt.app` / `0pt.dev`, 2026-08-11): båda svarar fortfarande med Vercel-nameservrar. |
|
||||
| **Bevis** | `nslookup -type=NS 0pt.app` och `0pt.dev` (körd denna session, 2026-08-11). |
|
||||
|
||||
---
|
||||
|
||||
## B-4 🔴 — `middleware.ts` exekverar inte i den liva Vercel-produktionsbygget
|
||||
|
||||
| Fält | Innehåll |
|
||||
|---|---|
|
||||
| **Vad blockerar** | Trots att auth-koden i `middleware.ts` är fail-closed i källan, och trots att en färsk fix (PR #11, `fix(security): require auth on GET /api/orders and GET /api/admin/metrics`, commit `15a0dbe`) redan är mergad till `main`, svarar de liva produktionsrutterna fortfarande anonymt. **Verifierat färskt denna session**: `GET https://0pt.app/api/0pt`, `/api/orders` och `/api/admin/metrics` gav samtliga **HTTP 200** anonymt (2026-08-11). Detta är repots egen dokumenterade lucka P0-1/S1 i `BACKLOG.md`/`GOLDEN-STANDARD-REPORT.md` — rotorsaken är okänd. |
|
||||
| **Vad krävs för att lyfta den** | Någon med Vercel-deploy-access måste undersöka byggutmatningen (Next 16 + `src/`-trädet, ev. edge-runtime-konfiguration) för att fastställa varför `middleware.ts` inte körs i produktionsbygget, fixa det, och verifiera med en levande anonym request → 401 efter fix. |
|
||||
| **Vem beslutar** | Ingen produktbeslutsfråga — tekniskt rotorsaksarbete som kräver Vercel-projektåtkomst som denna session saknar. |
|
||||
| **Nuvarande status** | Fortfarande öppen, och **allvarligare än BACKLOG.md:s ursprungliga formulering antyder**: koden är nu fixad (PR #11 mergad) men produktionen speglar ändå inte koden — detta bekräftar att gapet är i bygg-/deploykonfigurationen, inte (bara) i applikationskoden. |
|
||||
| **Bevis** | `docs/compliance/BACKLOG.md` P0-1; `docs/compliance/GOLDEN-STANDARD-REPORT.md` C1/C3; `curl -o /dev/null -w '%{http_code}'` mot samtliga tre rutter (körd denna session, 2026-08-11, samtliga 200). |
|
||||
|
||||
---
|
||||
|
||||
## Undantag (formellt, oförändrat av detta dokument)
|
||||
|
||||
Detta dokument beskriver ovanstående blockerare **utan att lösa dem**. Ingen av
|
||||
posterna ovan har medfört ändringar i `services/scrapers/{waf_bypass,engine,engine_v2}.py`,
|
||||
`services/scrapers/requirements.txt`, `services/infra/docker-compose.data-stack.yml`,
|
||||
`docs/compliance/BACKLOG.md`, eller PR #7 (varken återöppnad eller mergad). Den frågan
|
||||
är explicit under hantering i ett separat, parallellt spår per uppdragets instruktion.
|
||||
@@ -56,7 +56,7 @@
|
||||
| D1 | Container icke-root, multi-stage, utan tsx i prod | ✅ | Dockerfile: node:24-alpine, builder+runtime, `USER node`, `CMD next start` (ingen tsx) |
|
||||
| D2 | /health verifierar beroenden | ⚠️ | /api/health rapporterar db/queue lägen men alltid `ok:true` — liveness ej readiness; "alltid grönt för en döende instans" (ARCHITECTURE §6) |
|
||||
| D3 | CI grön på main | ✅ | Gitea Actions run 3759 success; deploy.yml röd men avsiktligt (guard) |
|
||||
| D4 | Deploy till Coolify provisionerad | ❌ | Coolify-app ej provisionerad; deploy.yml placeholder-UUID (avsiktligt röd); node:null |
|
||||
| D4 | Deploy till Coolify provisionerad | ⚠️ **(uppdaterad 2026-09-06, var ❌)** | Coolify-app ÄR provisionerad: repo-variabeln `COOLIFY_APP_ID` innehåller en riktig 24-teckens-UUID och repo-secreten `COOLIFY_TOKEN` finns (skapad 2026-08-31). Guarden i `cd.yml` passerar och deployen körde `coolify deploy HTTP 200` på push `1ff34410`. Kvarstår ⚠️ och inte ✅: probe 2026-09-06 08:00 UTC visar att `0pt.app` fortfarande svarar från **Vercel** med **HTTP 402 `DEPLOYMENT_DISABLED`** — publika domänen är nere, DNS är inte omlagd, och Coolify-appen serverar ingen `0pt.app`-trafik. Att webhooken accepteras (HTTP 200) betyder inte att en container kommer upp frisk |
|
||||
| D5 | Backup + testad restore | ❌ | Ingen backup, ingen DB i prod, restore aldrig testat (ARCHITECTURE §6) |
|
||||
| D6 | Loggning/observabilitet | ⚠️ | Ingen strukturerad loggning/OTel; endast console.warn (ARCHITECTURE §6) |
|
||||
| D7 | Infisical-projekt + token tillgängligt för denna session | ➖ | **Blockerare:** ingen Infisical-token tillhandahållen — projekt/secret-bootstrap kan inte verifieras (se SLUTRAPPORT) |
|
||||
|
||||
@@ -114,7 +114,7 @@ v0-datahub, prt-* etc.) finns i 0pt.
|
||||
| /api/health | 200, `{"ok":true,"db":"not_configured","queue":"not_configured"}` | live-probe 2026-08-10 |
|
||||
| /api/0pt (anonym) | **200 anonymt** — middleware fail-closed verkar INTE köra i prod (lucka S1) | live-probe 2026-08-10 |
|
||||
| /api/orders (anonym) | **200 anonymt** — inget skyddat (lucka S3) | live-probe 2026-08-10 |
|
||||
| CI (Gitea Actions) | senaste ci.yml på main: **success** (run 3759, 2026-08-09). deploy.yml: röd men **avsiktligt** (guard: placeholder-app-ID). 48 runs totalt | Gitea API `/actions/runs` |
|
||||
| CI (Gitea Actions) | senaste ci.yml på main: **success** (run 3759, 2026-08-09). ~~deploy.yml: röd men **avsiktligt** (guard: placeholder-app-ID)~~ **korrigerat 2026-09-06:** cd.yml är **grön** och deployar på riktigt — `COOLIFY_APP_ID` är en riktig UUID, guarden passerar, `coolify deploy HTTP 200` (run 11695, jobb 26295, head `1ff34410`). 48 runs totalt vid ursprungsmätningen | Gitea API `/actions/runs`, `/actions/variables`, `/actions/secrets` |
|
||||
| Secrets | Inga hårdkodade hemligheter hittade (grep-skannat). **Infisical ej inkopplat** — inga referenser i repot | grep + ARCHITECTURE §5 |
|
||||
| Backup | Ingen (ingen DB i prod att backa upp) | ARCHITECTURE §6 |
|
||||
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
# SCAFFOLD-TODO — vad som är härlett och vad som måste verifieras
|
||||
|
||||
**Genererad:** 2026-08-11 av `siax-scaffold` (siax-standard,
|
||||
`packages/scaffold`) mot Golden Standard v2.0.
|
||||
|
||||
Den här filen finns för att en mall inte får låtsas veta saker. Varje rad nedan
|
||||
är antingen **härledd ur repot** (källan står i kolumn 3) eller ett
|
||||
**konservativt defaultvärde** som en människa måste bekräfta. Radera raden när
|
||||
den är verifierad — filen ska till slut kunna tas bort helt.
|
||||
|
||||
## Härledda värden i `siax.repo.json`
|
||||
|
||||
| Fält | Värde | Härlett ur |
|
||||
|---|---|---|
|
||||
| `repo` | `0PT` | `git remote.origin.url` (VERSALER per REPO-08) |
|
||||
| `codeowners` | `["@sax3l"]` | default (organisationens ägare) |
|
||||
| `category` | `platform` | befintlig `siax.repo.json` |
|
||||
| `subcategory` | `data-os` | befintlig `siax.repo.json` |
|
||||
| `operationalState` | `degraded` | driftsartefakter finns (Dockerfile + siax.config.json) — EJ VERIFIERAT driftläge |
|
||||
| `domains` | `0 st` | **RÄTTAT MANUELLT:** inte "inga funna" — den BEFINTLIGA `siax.repo.json` listar redan 2 domäner (`0pt.app`, `api.0pt.app`), men som STRÄNGAR. Det nya schemat kräver objekt, så `inherit()` kastar dem (ogiltig typ) och räknar 0 giltiga. Uppgiften är att konvertera de 2 befintliga domänerna till objektformat — inte att hitta på domäner från noll. |
|
||||
| `children` | `4 st` | faktiska kataloger under apps/, packages/, services/ |
|
||||
| `packageName` | `@siax/0pt` | befintlig `siax.repo.json` |
|
||||
|
||||
## Måste verifieras av människa
|
||||
|
||||
- [ ] Bekräfta `codeowners` — scaffolden defaultade till `@sax3l` (CI-14).
|
||||
- [ ] Rätta `operationalState` mot verkligheten (`docker ps` på noden, GOV-06). Scaffolden satte `degraded` eftersom driftsartefakter finns men driftläget inte går att mäta från filsystemet — det är det konservativa valet som HÅLLER REPO-06/CD-01/API-01 aktiva i doctorn i stället för att tysta dem. **OBS (manuellt tillagt):** `siax.repo.json` listar redan `domains: ["0pt.app", "api.0pt.app"]` och `deploy.target: "coolify"`, vilket tyder på ett repo som ÄR eller HAR VARIT driftsatt — bekräfta det faktiska läget mot noden i stället för att anta något åt endera hållet.
|
||||
- [ ] Beskriv varje post i `children[]` — sökvägarna är härledda ur filträdet, syftet är inte (ARK-11).
|
||||
- [ ] Konvertera de 2 befintliga domänerna (`0pt.app`, `api.0pt.app`) i `siax.repo.json` från strängar till objekt enligt det nya schemat — de finns redan, de bara har fel typ (REPO-04). Se raden `domains` ovan.
|
||||
- [ ] **RÄTTAT MANUELLT — gäller INTE detta repo:** den ursprungliga raden här löd "Fastställ `appId` mot CL0UD APP_REGISTRY och ta bort `appIdJustification` (REPO-05)". Det är en av scaffoldens FASTA todos som skrivs oavsett läge. Den BEFINTLIGA `siax.repo.json` har redan `appId: "0PT"` satt (inte `null`), och det finns inget `appIdJustification`-fält att ta bort — det fältet hade bara uppstått om scaffolden skrivit en NY deskriptor, vilket AR-1 förhindrade här eftersom filen redan fanns. Kvarstående uppgift: bekräfta att `0PT` faktiskt är registrerat i CL0UD APP_REGISTRY.
|
||||
- [ ] Fyll i `consumedBy[]` i `siax.repo.json` från FAS 2-beroendekartan — det fältet SAKNAS helt idag. (`consumes[]` är däremot redan ifyllt i den befintliga filen: `CL0UD, ST0RE, N0D, AUD0, INF0` — ursprungsraden här bad felaktigt om att fylla i båda som om ingetdera fanns.)
|
||||
- [ ] Fyll i `node` (server5–server11) i `siax.repo.json`, eller lämna `null` om repot inte driftsätts.
|
||||
- [ ] Kör återställningen skarpt och skriv in datumet i `docs/runbook/backup-restore.md` (DOC-07 — den enda av de tio kontrollerna scaffolden inte kan stänga).
|
||||
- [ ] Rätta den BEFINTLIGA `siax.repo.json` — den validerar inte mot schemat (REPO-03/04) och scaffolden skriver inte över den (AR-1): (roten): obligatoriskt fält saknas: repo; (roten): obligatoriskt fält saknas: lifecycle; (roten): obligatoriskt fält saknas: gitHost; (roten): obligatoriskt fält saknas: consumedBy; (roten): obligatoriskt fält saknas: children; (roten): obligatoriskt fält saknas: operationalState; (roten): okänt fält "name" (additionalProperties: false); /domains/0: typen är string, förväntad object; /domains/1: typen är string, förväntad object; (roten): okänt fält "deploy" (additionalProperties: false); /designSystem: typen är null, förväntad string
|
||||
|
||||
## Kontroller som scaffolden AVSIKTLIGT lämnar röda
|
||||
|
||||
- **DOC-07** — `docs/runbook/backup-restore.md` saknar datum för bevisad
|
||||
restore. Scaffolden skriver aldrig in ett datum: kontrollen letar efter ett
|
||||
datum, och ett genererat sådant skulle göra den grön utan att någon
|
||||
återställning ägt rum. Kör återställningen, skriv in datumet.
|
||||
- **REPO-05** — **RÄTTAT MANUELLT, gäller INTE detta repo:** mallraden
|
||||
påstod "`appId` är `null` med motivering", men den BEFINTLIGA
|
||||
`siax.repo.json` har redan `appId: "0PT"` satt och inget
|
||||
`appIdJustification`-fält att ta bort. Det påståendet beskrev scaffoldens
|
||||
egen, aldrig skrivna, hypotetiska deskriptor (som hade defaultat till
|
||||
`null` om appId saknats) — inte vad som faktiskt ligger på disk. Se
|
||||
motsvarande punkt under "Måste verifieras av människa" ovan.
|
||||
- **API-02/03/04** — ett genererat kontraktsskelett beskriver ingen route.
|
||||
**OBS:** detta repo har redan `contracts/openapi.yaml` på plats (API-01
|
||||
PASS) — inget kontraktsskelett genererades här (action: `ok`, inte
|
||||
`create`); den här raden är generisk mallinformation om vad scaffolden
|
||||
gör NÄR den genererar ett skelett, inte ett påstående om att den gjorde
|
||||
det i detta repo.
|
||||
|
||||
---
|
||||
|
||||
_Genererad av `siax-scaffold` (siax-standard). Varje `TODO:` i filen måste
|
||||
fyllas i av en människa. Samlad lista: `docs/compliance/SCAFFOLD-TODO.md`._
|
||||
@@ -0,0 +1,150 @@
|
||||
# P6 guides — 0pt
|
||||
|
||||
BACKLOG P6: 19 GUIDE-level items, all marked S (small effort), all "NU"
|
||||
(safe to write now). Several have their own concrete artifact instead of a
|
||||
guide entry here — see the table. This file covers the rest: real,
|
||||
actionable "how to build this when it's prioritized" guidance grounded in
|
||||
0pt's actual code, not generic advice that could apply to any repo.
|
||||
|
||||
| Control | Item | Where |
|
||||
|---|---|---|
|
||||
| REPO-13 | `.editorconfig` | [`.editorconfig`](../../.editorconfig) |
|
||||
| ARK-14 | dependency-cruiser | [`.dependency-cruiser.cjs`](../../.dependency-cruiser.cjs), `pnpm run deps:check` |
|
||||
| API-24 | deprecation policy | [`docs/api/DEPRECATION_POLICY.md`](../api/DEPRECATION_POLICY.md) |
|
||||
| CD-23 | K8s manifest | [`deploy/k8s/`](../../deploy/k8s/) (reference only, not the deploy path) |
|
||||
| CI-16 | `act` | [`.actrc`](../../.actrc) |
|
||||
| DX-11 | Makefile | [`Makefile`](../../Makefile) |
|
||||
| DX-12 | onboarding | [`ONBOARDING.md`](../../ONBOARDING.md) |
|
||||
| CLI-13 | `doctor` CLI subcommand | see §1 below |
|
||||
| DOC-18 | documentation site | see §2 below |
|
||||
| UI-30 | PWA | see §3 below |
|
||||
| IAM-19 | impersonation | see §4 below |
|
||||
| PAY-11 | Hyperswitch ADR | see §5 below |
|
||||
| AI-18 | assistant | see §6 below |
|
||||
| OBS-21 | status page | see §7 below |
|
||||
| TEST-20 | chaos testing | see §8 below |
|
||||
| NET-19 | IPv6 | see §9 below |
|
||||
| SCALE-13 | read replicas | see §10 below |
|
||||
| OSS-10 | upstream contribution | see §11 below |
|
||||
| AGENT-13 | nightly agent | see §12 below |
|
||||
|
||||
---
|
||||
|
||||
## 1. CLI-13 — `doctor` CLI subcommand
|
||||
|
||||
There is no CLI surface at all yet (BACKLOG P1-5 — L effort, not attempted
|
||||
this pass). `pnpm run siax:doctor` / `make doctor`
|
||||
(`scripts/siax-doctor.mjs`) already covers the underlying check logic
|
||||
(AGENT-04). When P1-5 builds `apps/cli`, wire a `siax 0pt doctor` subcommand
|
||||
that shells out to (or re-imports) `scripts/siax-doctor.mjs` rather than
|
||||
re-implementing the checks — one source of truth for "is this repo
|
||||
compliant", not two that can drift.
|
||||
|
||||
## 2. DOC-18 — documentation site
|
||||
|
||||
Not built. When prioritized: `contracts/openapi.yaml` already exists as a
|
||||
real OpenAPI 3.1 SSOT, so the lowest-effort path is a static OpenAPI
|
||||
renderer (e.g. Redocly CLI or Scalar) pointed at that file, deployed as a
|
||||
static site — not a hand-authored docs site that would drift from the
|
||||
contract. `docs/*.md` could be published alongside via any static-site
|
||||
generator that reads plain Markdown (Docusaurus, Starlight); pick whichever
|
||||
matches what other SIAX Cloud repos already use, per BACKLOG's "one tool per
|
||||
job" rule — check `cl0ud`/`n0d` first before introducing a new one here.
|
||||
|
||||
## 3. UI-30 — PWA
|
||||
|
||||
Not built. Would need: a `manifest.json` (icons already exist at
|
||||
`public/icon*.png`/`icon.svg` — reuse them, don't regenerate), a service
|
||||
worker (Next.js has no built-in PWA plugin as of this repo's Next 16; a
|
||||
minimal hand-written SW covering `/api/health`-style offline detection would
|
||||
be more honest than a cargo-culted `next-pwa` config, given this app's
|
||||
`ignoreBuildErrors: true` posture already has enough unverified surface). Low
|
||||
priority — 0pt's UI is transactional (order flow, portal), not the kind of
|
||||
app users install to a home screen.
|
||||
|
||||
## 4. IAM-19 — impersonation
|
||||
|
||||
Not built. Needed pattern once an admin portal exists with real customer
|
||||
data: a support/admin session that can view-as a specific tenant/user,
|
||||
**always** logged to AUD0 (BACKLOG P0-18 — AUD0 client exists but is
|
||||
currently no-op without `AUD0_URL`) with the impersonator's real identity
|
||||
attached, never silently. Do not build a "become this user" auth shortcut
|
||||
without that audit trail landing first — an impersonation feature with no
|
||||
audit log is a privilege-escalation bug wearing a support-tool costume.
|
||||
|
||||
## 5. PAY-11 — Hyperswitch ADR
|
||||
|
||||
**Proposed, not accepted — payments provider is explicitly gated on the
|
||||
account owner's decision (BACKLOG P1-23).** This entry exists so the option
|
||||
is on record, not to decide it. Hyperswitch is an open-source payment
|
||||
orchestration layer (self-hostable) that could sit in front of
|
||||
Stripe/Revolut/others rather than 0pt integrating each provider directly
|
||||
(`lib/stripe.ts` today). Trade-off to weigh when P1-23 is actually decided:
|
||||
self-hosting Hyperswitch adds an operational dependency (another service to
|
||||
run/monitor) in exchange for provider-agnosticism; direct integration
|
||||
(current state) is simpler but couples this repo to Stripe's specific API
|
||||
shape. Not evaluated further here — that's the ADR this guide is explicitly
|
||||
not writing.
|
||||
|
||||
## 6. AI-18 — assistant
|
||||
|
||||
Not built. `app/api/ai/parse` exists today as a heuristic (no model call,
|
||||
per `CLAUDE.md`/`docs/02-architecture.md`'s "honest heuristic" framing) — a
|
||||
real assistant feature should route through INF0 (the fleet AI gateway,
|
||||
`INF0_URL` — documented target, not yet read by any code in this repo) per
|
||||
the platform mandate, never a raw provider key committed here.
|
||||
|
||||
## 7. OBS-21 — status page
|
||||
|
||||
Not built. Lowest-effort honest option once `/api/health` + `/api/ready`
|
||||
are actually monitored (BACKLOG P1-11, not yet wired to Prometheus/GlitchTip
|
||||
per that item): a self-hosted status page (e.g. Upptime, which is
|
||||
just-a-GitHub/Gitea-Actions-cron + static site, no server to run) polling
|
||||
those two endpoints. Do not stand up a status page before the endpoints it
|
||||
would display are meaningfully monitored — an unmonitored status page that
|
||||
says "operational" is worse than no status page.
|
||||
|
||||
## 8. TEST-20 — chaos testing
|
||||
|
||||
Not built, and low-value until BACKLOG P0-12 lands (real DB/persistence —
|
||||
right now there's no durable state to chaos-test; killing the process loses
|
||||
nothing that wasn't already going to be lost on the next cold start). Once
|
||||
persistence exists: start with the cheapest chaos test that matches 0pt's
|
||||
actual documented failure mode — kill the Postgres connection mid-request
|
||||
and assert `/api/ready` flips to 503 within one health-check interval (this
|
||||
is now directly testable thanks to BACKLOG P0-3's real dependency probes).
|
||||
|
||||
## 9. NET-19 — IPv6
|
||||
|
||||
Not evaluated. `docs/architecture/ARCHITECTURE.md` doesn't currently record
|
||||
whether the Coolify target node(s) or `0pt.app`'s DNS have AAAA records —
|
||||
check `z0ne` (SIAX's own DNS product) once a real Coolify app exists for
|
||||
this repo; premature to configure before there's a fleet IP to point at.
|
||||
|
||||
## 10. SCALE-13 — read replicas
|
||||
|
||||
Not applicable yet — there is no primary database in production to
|
||||
replicate (BACKLOG P0-12: state lives in in-memory `Map`s, zero DB
|
||||
provisioned). Revisit once P0-12 lands and real read/write load is
|
||||
measured; provisioning a replica against a database that doesn't exist yet
|
||||
would be pure speculation.
|
||||
|
||||
## 11. OSS-10 — upstream contribution
|
||||
|
||||
No dependency in this repo has been identified as needing a patch upstream
|
||||
rather than a workaround here (checked: no local patches in
|
||||
`pnpm-lock.yaml`'s `patchedDependencies`, none found). Nothing to
|
||||
contribute upstream today; revisit if/when a real workaround is added for
|
||||
an upstream bug — file the issue/PR there instead of carrying the
|
||||
workaround silently forever.
|
||||
|
||||
## 12. AGENT-13 — nightly agent
|
||||
|
||||
`nightly.yml` already re-runs the full quality + (now blocking) security
|
||||
gate daily (BACKLOG P0-16). A step beyond that — an agent that reads the
|
||||
nightly failure and opens a triage PR — is AGENT-06 territory (BACKLOG P1-19
|
||||
already scopes AGENT-06 as not attempted this pass: it needs real
|
||||
OpenHands/Aider/SWE-agent infrastructure via INF0, which doesn't exist yet
|
||||
for this repo). Do not build a bespoke nightly-agent script here ahead of
|
||||
that shared infrastructure landing — it would be exactly the kind of
|
||||
one-off tool the "one tool per job" rule (`CLAUDE.md` §4) warns against.
|
||||
@@ -1,17 +1,76 @@
|
||||
# Runbook: Backup & Restore — 0pt
|
||||
|
||||
> Status 2026-08-10: **ingen databas i produktion** (`/api/health` →
|
||||
> `db: not_configured`); allt state är in-memory (försvinner vid kallstart).
|
||||
> Denna runbook beskriver läget idag + målbilden när Postgres provisioneras.
|
||||
> Status 2026-08-10 (still true as of this P014 pass, not independently
|
||||
> re-verified against a live deployment): **ingen databas i produktion**
|
||||
> (`/api/health` → `db: not_configured`); allt state är in-memory (försvinner
|
||||
> vid kallstart). Denna runbook beskriver läget idag + målbilden när Postgres
|
||||
> provisioneras. **Vad SOM ändrades i P014**: migreringskedjan
|
||||
> (`sql/001` → `006`) gick tidigare inte att köra alls end-to-end mot en
|
||||
> tom databas (se "P014 restore-övning" nedan) — det är nu fixat och
|
||||
> verifierat, oberoende av om produktion faktiskt har `DATABASE_URL` satt.
|
||||
|
||||
## Nuläge
|
||||
|
||||
- Ingen backup behövs/sker idag — det finns inget att backa upp (in-memory).
|
||||
- `sql/001_init.sql` är schemat (13 tabeller + pgvector/pgcrypto).
|
||||
- `sql/002_source_registry.sql` — Source Registry-tabellerna.
|
||||
- `sql/003_datasets.sql` (P014) — `opt_datasets`, tidigare saknad helt (se nedan).
|
||||
- `sql/004/005` är iceberg-bärande tabeller.
|
||||
- `sql/006_orders_idempotency.sql` (P014) — `orders.idempotency_key` + unikt index.
|
||||
- **Kända blockerare:** estate-som-helhet saknar offsite-backup (Hetzner
|
||||
Robot-creds). Denna repo kan inte lösa det enskilt.
|
||||
|
||||
## P014 restore-övning: EXEKVERAD 2026-09-03 (verklig, mätt, inte prosa)
|
||||
|
||||
Körd mot verklig, engångs, isolerad infrastruktur (Docker, `pgvector/pgvector:pg16`
|
||||
— samma image som `compose.dev.yaml`/`infra/docker-compose.yml`) — inte simulerad.
|
||||
Ingen produktionsdata, inga persistenta artefakter kvarlämnade.
|
||||
|
||||
**Steg 1 — upptäckt: migreringskedjan gick inte att köra.** Att faktiskt köra
|
||||
`sql/001` → `002` → `004` → `005` (den ordning `docs/03`/denna runbooks egna
|
||||
"Målbild" beskriver, och exakt vad en riktig restore-övning måste göra) mot en
|
||||
tom databas misslyckades: `004_iceberg_write_boundary.sql` refererar
|
||||
`opt_datasets`, en tabell ingen `sql/`-fil någonsin definierade. Filens egen
|
||||
kommentar hade redan flaggat detta ("Flagging, not fixing here... remains
|
||||
open") men ingen hade tidigare faktiskt kört kedjan för att bekräfta felet.
|
||||
**Fixat**: `sql/003_datasets.sql` tillagd (matchar
|
||||
`src/services/dataset-service.ts`s `DatasetRecord`-form exakt, inte en
|
||||
gissning) — kedjan `001→002→003→004→005→006` kör nu rent mot en tom databas.
|
||||
|
||||
**Steg 2 — upptäckt: `lib/db.ts` kunde aldrig ansluta till en icke-TLS
|
||||
Postgres.** `getSql()` hårdkodade `ssl: "require"` ovillkorligt — vilket gjorde
|
||||
att `compose.dev.yaml`s egen lokala Postgres-container (ingen TLS)
|
||||
ALDRIG hade kunnat nås via denna funktion, konsekvent med den filens egen
|
||||
"UNVERIFIED... not smoke-tested"-flagga. **Fixat**: `DATABASE_SSL=false` är
|
||||
nu ett explicit opt-out (standard oförändrad — `"require"` — för varje verklig
|
||||
driftsättning).
|
||||
|
||||
**Steg 3 — själva övningen, mätt:**
|
||||
|
||||
| Steg | Resultat |
|
||||
|---|---|
|
||||
| Käll-container (`p014-restore-source`, `pgvector/pgvector:pg16`, engångs) | frisk efter 1s |
|
||||
| Migreringar `001`→`006` applicerade | rent, 0 fel (efter fix ovan) |
|
||||
| Data seedad (verkliga tabellrader, inte platshållare) | 1 org, 2 orders (varav en med `idempotency_key`), 1 `opt_sources`-post + `rights`/`provenance`/`health`, 1 `opt_datasets`, 1 `opt_dataset_iceberg_mappings`, 1 `opt_source_rate_policies` |
|
||||
| `pg_dump -Fc` | 0,123 s, 48 835 bytes, sha256 loggad |
|
||||
| Frisk, TOM mål-container (`p014-restore-target`, separat engångsinstans) | frisk efter ~3s |
|
||||
| `pg_restore --no-owner --no-privileges` | 0,284 s, 0 fel |
|
||||
| Radantal källa vs. mål, samtliga 9 tabeller | **identiska** |
|
||||
| `md5`-checksumma på hela `orders`-tabellens innehåll (källa vs. mål) | **identisk** (`95f7d0f9…`) |
|
||||
| `md5`-checksumma på hela `opt_sources`-tabellens innehåll (källa vs. mål) | **identisk** (`b4612dbf…`) |
|
||||
| Verklig applikationskod mot den RESTORERADE databasen: `lib/orders-store.ts` (`getOrder`, `findOrderByIdempotencyKey`) | läser tillbaka exakt rätt `config`/`total_ore`/`est_confidence`/idempotency-nyckel |
|
||||
| Verklig applikationskod mot den RESTORERADE databasen: `SourceRegistryService` (riktigt Postgres-repository, `src/repositories/postgres-source-registry-repository.ts`) | `describeSource` returnerar korrekt `name`/`rights.license_id`/`provenance.method`/`health.status`; tvärtenant-isolering (`NOT_FOUND`, ingen existence-oracle) håller fortfarande mot den restorerade databasen |
|
||||
|
||||
Slutsats: en klon av denna schema+data-modell kan verkligen provisioneras,
|
||||
backas upp och återställas end-to-end idag — det kunde den INTE innan denna
|
||||
övning (migreringskedjan var trasig). Engångscontainrarna revs efter
|
||||
övningen; ingen data eller infrastruktur kvarlämnad.
|
||||
|
||||
**Kvarstående, inte löst av denna övning:** produktionens faktiska
|
||||
backup-schema/retention/offsite-kopia (se "Målbild" nedan — fortfarande
|
||||
målbild, inte verifierad mot en live driftsättning i denna pass) och
|
||||
Hetzner-blockeraren för estate-bred offsite-backup.
|
||||
|
||||
## Målbild (när `DATABASE_URL` satts)
|
||||
|
||||
### Daglig backup (P2-1 i BACKLOG)
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|---|---|---|
|
||||
| `/api/health` grön men `/api/orders` tom | Ingen DB i prod (in-memory-store) | Sätt `DATABASE_URL`; se BACKLOG P3-4 |
|
||||
| `/api/0pt/*` svarar 200 anonymt | Middleware körs inte på Vercel-deployen (lucka S1) | Rotorsaksanalys; verifiera 401 anonymt före "skyddad" |
|
||||
| deploy.yml-röd på main | Guard: placeholder `COOLIFY_APP_ID` (avsiktligt) | Provisionera Coolify-app + fyll UUID |
|
||||
| ~~deploy.yml-röd på main~~ | ~~Guard: placeholder `COOLIFY_APP_ID` (avsiktligt)~~ **INAKTUELL 2026-09-06** — guarden passerar numera; `COOLIFY_APP_ID` är en riktig UUID och `COOLIFY_TOKEN` finns | Rätt symtom i dag är det omvända: **varje merge till `main` utlöser en riktig omdeploy** mot cloud.siax.io (verifierat: push `1ff34410`, jobb `Deploy to Coolify / deploy` = success, `coolify deploy HTTP 200`). Vet vad du deployar innan du mergar |
|
||||
| Test `*.realinfra.test.ts` skip | ST0RE inte nåbar från CI | Inte en incident — förväntat beteende |
|
||||
| Trivy-fynd på `services/scrapers/*` | Medvetet behållen bypass-stack | Läs GOLDEN-STANDARD-REPORT C6; triage, radera ej |
|
||||
|
||||
|
||||
+2
-1
@@ -7,7 +7,8 @@
|
||||
|
||||
| SLO | Mål | Nuläge (2026-08-10) |
|
||||
|---|---|---|
|
||||
| `/api/health` (liveness) | 99,9 % | ✅ 200 (men alltid `ok:true` — readiness saknas, se D2) |
|
||||
| `/api/health` (liveness) | 99,9 % | ✅ 200; `ok` speglar nu en riktig per-dependency-probe (fixat BACKLOG P0-3) |
|
||||
| `/api/ready` (readiness) | 99,5 % | ✅ ny (BACKLOG P0-3) — 503 när `db` inte är konfigurerad+nåbar |
|
||||
| Orderflöde `/api/orders` | 99,5 % | ⚠️ fungerar utan DB (in-memory), data försvinner vid restart |
|
||||
| Capability-API `/api/0pt/*` | 99,5 % | ⚠️ live 200 anonymt (lucka S1) — målbild: 401 utan token |
|
||||
| Engine (FastAPI, när deployad) | 99,0 % | ej deployad (EJ VERIFIERAT) |
|
||||
|
||||
@@ -13,11 +13,30 @@ export function hasDb(): boolean {
|
||||
return Boolean(dbUrl());
|
||||
}
|
||||
|
||||
/**
|
||||
* P014 fix: `ssl` used to be hardcoded to `"require"` unconditionally. That
|
||||
* is the right default for every REAL deployment target this app has
|
||||
* (Vercel Postgres, managed Postgres providers), but it made `compose.dev.yaml`'s
|
||||
* local Postgres container -- which speaks plain, non-TLS Postgres -- and
|
||||
* ANY other throwaway/local Postgres (e.g. a restore-drill target) silently
|
||||
* unreachable via this function, contradicting that file's own doc comment
|
||||
* ("local-only Postgres for developers who want real persistence") and its
|
||||
* own "UNVERIFIED... not smoke-tested end to end" caveat -- this is
|
||||
* precisely why it was never smoke-tested: it could not have worked.
|
||||
* `DATABASE_SSL` is opt-in-to-disable ONLY; absence (the default, matching
|
||||
* every existing deployment's actual env) keeps the exact prior behavior.
|
||||
*/
|
||||
export function sslMode(): "require" | false {
|
||||
const raw = process.env.DATABASE_SSL?.trim().toLowerCase();
|
||||
if (raw === "false" || raw === "disable" || raw === "off" || raw === "0") return false;
|
||||
return "require";
|
||||
}
|
||||
|
||||
export function getSql() {
|
||||
const url = dbUrl();
|
||||
if (!url) return null;
|
||||
if (!_sql) {
|
||||
_sql = postgres(url, { prepare: false, idle_timeout: 20, max: 5, ssl: "require" });
|
||||
_sql = postgres(url, { prepare: false, idle_timeout: 20, max: 5, ssl: sslMode() });
|
||||
}
|
||||
return _sql;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
// Dependency health probes shared by GET /api/health (liveness) and
|
||||
// GET /api/ready (readiness). See docs/architecture/ARCHITECTURE.md
|
||||
// "Hälsokontroll" and BACKLOG P0-3 for the finding this fixes: the old
|
||||
// /api/health handler only ever checked *presence* of an env var and
|
||||
// hardcoded `ok: true` regardless of whether anything was actually
|
||||
// reachable -- "an uptime monitor watching HTTP 200 would show green for a
|
||||
// dying instance". Each check below returns one of four honest states:
|
||||
//
|
||||
// "ok" -- configured AND a live round trip succeeded.
|
||||
// "error" -- configured but the round trip failed (bad creds,
|
||||
// host unreachable, timeout, ...). detail carries the
|
||||
// underlying error message.
|
||||
// "not_configured" -- no env var set. Not a failure by itself (0pt is
|
||||
// designed to build/boot with everything unset).
|
||||
// "not_integrated" -- env var IS set, but no client code in this repo
|
||||
// actually uses it (currently only REDIS_URL/queue --
|
||||
// see .env.example). Reporting "ok" here would be a
|
||||
// fabricated check; reporting "not_configured" would
|
||||
// hide that a value was provided. This is its own
|
||||
// honest state rather than either of those two.
|
||||
import { dbUrl, getSql } from "./db";
|
||||
import { getStripe } from "./stripe";
|
||||
import { hasEngine } from "./engine";
|
||||
|
||||
export type DepStatus = "ok" | "error" | "not_configured" | "not_integrated";
|
||||
|
||||
export interface DepResult {
|
||||
status: DepStatus;
|
||||
detail?: string;
|
||||
latencyMs?: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_TIMEOUT_MS = 2000;
|
||||
|
||||
function errMessage(err: unknown): string {
|
||||
return err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
|
||||
function withTimeout<T>(promise: Promise<T>, ms: number, label: string): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
|
||||
promise.then(
|
||||
(v) => {
|
||||
clearTimeout(timer);
|
||||
resolve(v);
|
||||
},
|
||||
(e) => {
|
||||
clearTimeout(timer);
|
||||
reject(e);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
// Minimal shape of the one call site needs from the postgres client --
|
||||
// matches `postgres`'s tagged-template callable, but kept narrow so tests
|
||||
// can inject a fake without constructing a real `postgres.Sql`.
|
||||
export type SqlPing = (strings: TemplateStringsArray, ...values: unknown[]) => Promise<unknown>;
|
||||
|
||||
/**
|
||||
* Postgres reachability. `select 1` is the standard zero-cost liveness query
|
||||
* -- it never touches application tables, so it can't fail on schema drift.
|
||||
*/
|
||||
export async function checkDatabase(
|
||||
sqlClient: SqlPing | null = getSql() as unknown as SqlPing | null,
|
||||
timeoutMs: number = DEFAULT_TIMEOUT_MS,
|
||||
): Promise<DepResult> {
|
||||
if (!dbUrl() || !sqlClient) return { status: "not_configured" };
|
||||
const start = Date.now();
|
||||
try {
|
||||
await withTimeout(sqlClient`select 1`, timeoutMs, "database ping");
|
||||
return { status: "ok", latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return { status: "error", detail: errMessage(err), latencyMs: Date.now() - start };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* FastAPI data engine (services/api) reachability via its own `GET /health`
|
||||
* (services/api/orchestrator/main.py). Uses a real HTTP round trip, not just
|
||||
* `hasEngine()` presence -- ENGINE_URL pointing at a dead host used to read
|
||||
* as healthy.
|
||||
*/
|
||||
export async function checkEngine(
|
||||
fetchImpl: typeof fetch = fetch,
|
||||
timeoutMs: number = DEFAULT_TIMEOUT_MS,
|
||||
): Promise<DepResult> {
|
||||
if (!hasEngine()) return { status: "not_configured" };
|
||||
const base = process.env.ENGINE_URL as string;
|
||||
const start = Date.now();
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const res = await fetchImpl(`${base}/health`, {
|
||||
signal: controller.signal,
|
||||
cache: "no-store",
|
||||
headers: { "X-Internal-Token": process.env.INTERNAL_TOKEN ?? "" },
|
||||
});
|
||||
if (!res.ok) {
|
||||
return { status: "error", detail: `HTTP ${res.status}`, latencyMs: Date.now() - start };
|
||||
}
|
||||
return { status: "ok", latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return { status: "error", detail: errMessage(err), latencyMs: Date.now() - start };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
// Minimal shape of the one Stripe call this needs -- matches
|
||||
// `Stripe#balance.retrieve`, kept narrow so tests can inject a fake instead
|
||||
// of constructing a real Stripe client.
|
||||
export interface StripeBalanceClient {
|
||||
balance: { retrieve(): Promise<unknown> };
|
||||
}
|
||||
|
||||
/**
|
||||
* Stripe reachability. `balance.retrieve()` is Stripe's documented
|
||||
* lightweight "is this key valid and can I reach the API" call -- it makes
|
||||
* one real API round trip (catching e.g. a revoked/placeholder key) without
|
||||
* mutating anything or touching a specific resource.
|
||||
*/
|
||||
export async function checkStripe(
|
||||
client: StripeBalanceClient | null = getStripe() as unknown as StripeBalanceClient | null,
|
||||
timeoutMs: number = DEFAULT_TIMEOUT_MS,
|
||||
): Promise<DepResult> {
|
||||
if (!client) return { status: "not_configured" };
|
||||
const start = Date.now();
|
||||
try {
|
||||
await withTimeout(client.balance.retrieve(), timeoutMs, "stripe balance.retrieve");
|
||||
return { status: "ok", latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return { status: "error", detail: errMessage(err), latencyMs: Date.now() - start };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Redis/queue: presence-only, honestly. No queue client library is a
|
||||
* dependency of this package (grep-verified against package.json) and no
|
||||
* code path in this repo reads REDIS_URL to open a connection -- so there is
|
||||
* no real check to run. Reporting "ok" would fabricate a probe that doesn't
|
||||
* exist; this reports "not_integrated" instead of silently upgrading to a
|
||||
* pretend pass.
|
||||
*/
|
||||
export function checkQueue(): DepResult {
|
||||
return { status: process.env.REDIS_URL ? "not_integrated" : "not_configured" };
|
||||
}
|
||||
|
||||
export interface DependencyResults {
|
||||
db: DepResult;
|
||||
engine: DepResult;
|
||||
stripe: DepResult;
|
||||
queue: DepResult;
|
||||
}
|
||||
|
||||
export async function checkAllDependencies(): Promise<DependencyResults> {
|
||||
const [db, engine, stripe] = await Promise.all([checkDatabase(), checkEngine(), checkStripe()]);
|
||||
return { db, engine, stripe, queue: checkQueue() };
|
||||
}
|
||||
|
||||
/** True when any dependency that IS configured/integrated is broken. */
|
||||
export function hasDegradedDependency(deps: DependencyResults): boolean {
|
||||
return Object.values(deps).some((d) => d.status === "error");
|
||||
}
|
||||
|
||||
/**
|
||||
* Readiness (unlike liveness) treats "not configured" for the deps this app
|
||||
* cannot serve real traffic without as not-ready too. Only `db` is load-bearing
|
||||
* today -- engine/stripe/queue are optional integrations the app is designed
|
||||
* to run without (see .env.example).
|
||||
*/
|
||||
export function isReady(deps: DependencyResults): boolean {
|
||||
return deps.db.status === "ok";
|
||||
}
|
||||
+97
-12
@@ -29,6 +29,45 @@ export interface Order {
|
||||
|
||||
// Non-durable fallback used only until a database is provisioned.
|
||||
const mem = new Map<string, Order>();
|
||||
// idempotency_key -> order_id, mirroring the Postgres unique index added by
|
||||
// sql/006_orders_idempotency.sql for the in-memory fallback path.
|
||||
const memIdempotencyIndex = new Map<string, string>();
|
||||
|
||||
/**
|
||||
* Thrown by {@link createOrder} when a caller reuses an `Idempotency-Key`
|
||||
* for a semantically different request (different `flow` and/or `config`)
|
||||
* than the one that key was first used with. A route handler should map
|
||||
* this to HTTP 409 -- silently returning either order would be wrong (the
|
||||
* caller asked for the SAME operation to be safely retried, not for two
|
||||
* different operations to collapse into one).
|
||||
*/
|
||||
export class IdempotencyKeyConflictError extends Error {
|
||||
constructor(public readonly idempotencyKey: string) {
|
||||
super(`Idempotency-Key "${idempotencyKey}" was already used to create a different order (different flow/config)`);
|
||||
this.name = "IdempotencyKeyConflictError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Postgres error code 23505 = unique_violation (postgres.js surfaces it as `.code`). */
|
||||
function isUniqueViolation(err: unknown): boolean {
|
||||
return typeof err === "object" && err !== null && (err as { code?: unknown }).code === "23505";
|
||||
}
|
||||
|
||||
function sameSubmission(a: Order, flow: string, config: OrderConfig): boolean {
|
||||
return a.flow === flow && JSON.stringify(a.config) === JSON.stringify(config);
|
||||
}
|
||||
|
||||
export async function findOrderByIdempotencyKey(key: string): Promise<Order | null> {
|
||||
const sql = getSql();
|
||||
if (sql) {
|
||||
const rows = await sql<Order[]>`
|
||||
SELECT order_id, flow, status, config, total_ore, deposit_ore, balance_ore, est_days, est_confidence, created_at
|
||||
FROM orders WHERE idempotency_key = ${key} LIMIT 1`;
|
||||
return rows[0] ? coerceOrder(rows[0]) : null;
|
||||
}
|
||||
const id = memIdempotencyIndex.get(key);
|
||||
return id ? mem.get(id) ?? null : null;
|
||||
}
|
||||
|
||||
// postgres.js (pinned v3.4.9) returns bigint (total_ore/deposit_ore/balance_ore)
|
||||
// and numeric (est_confidence) columns as strings by default, not JS numbers.
|
||||
@@ -58,21 +97,66 @@ function estimateFor(config: OrderConfig) {
|
||||
});
|
||||
}
|
||||
|
||||
export async function createOrder(flow: string, config: OrderConfig): Promise<Order> {
|
||||
export interface CreateOrderResult {
|
||||
order: Order;
|
||||
/** True when this call returned a PRE-EXISTING order (a safe retry under the same Idempotency-Key), not a newly created one. */
|
||||
replayed: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* P014: `idempotencyKey` is optional -- a caller that omits it (or the
|
||||
* pre-P014 call shape) behaves exactly as before, creating a fresh row every
|
||||
* call. When supplied (the `Idempotency-Key` header convention, see
|
||||
* sql/006_orders_idempotency.sql), a retry with the SAME key returns the
|
||||
* original order unchanged (`replayed: true`) instead of creating a
|
||||
* duplicate; a retry with the same key but a DIFFERENT flow/config throws
|
||||
* {@link IdempotencyKeyConflictError} rather than silently picking one.
|
||||
*
|
||||
* The Postgres path checks-then-inserts, which has a real (if narrow) race
|
||||
* window between two concurrent requests carrying the same fresh key -- the
|
||||
* UNIQUE INDEX from sql/006 is the actual source of truth, and a
|
||||
* unique-violation on INSERT is caught and re-resolved the same way a
|
||||
* pre-existing row would be, rather than surfacing a raw DB error.
|
||||
*/
|
||||
export async function createOrder(flow: string, config: OrderConfig, idempotencyKey?: string): Promise<CreateOrderResult> {
|
||||
const key = idempotencyKey?.trim() || undefined;
|
||||
if (key) {
|
||||
const existing = await findOrderByIdempotencyKey(key);
|
||||
if (existing) {
|
||||
if (sameSubmission(existing, flow, config)) return { order: existing, replayed: true };
|
||||
throw new IdempotencyKeyConflictError(key);
|
||||
}
|
||||
}
|
||||
|
||||
const est = estimateFor(config);
|
||||
const sql = getSql();
|
||||
if (sql) {
|
||||
const [row] = await sql<{ order_id: string; created_at: string }[]>`
|
||||
INSERT INTO orders (flow, status, config, total_ore, deposit_ore, balance_ore, est_days, est_confidence)
|
||||
VALUES (${flow}, 'draft', ${sql.json(config as Parameters<typeof sql.json>[0])}, ${est.price * 100}, ${est.deposit * 100},
|
||||
${est.remainder * 100}, ${est.timelineDays}, ${est.estConfidence / 100})
|
||||
RETURNING order_id, created_at`;
|
||||
return {
|
||||
order_id: row.order_id, flow, status: "draft", config,
|
||||
total_ore: est.price * 100, deposit_ore: est.deposit * 100, balance_ore: est.remainder * 100,
|
||||
est_days: est.timelineDays, est_confidence: est.estConfidence / 100, created_at: row.created_at,
|
||||
};
|
||||
try {
|
||||
const [row] = await sql<{ order_id: string; created_at: string }[]>`
|
||||
INSERT INTO orders (flow, status, config, total_ore, deposit_ore, balance_ore, est_days, est_confidence, idempotency_key)
|
||||
VALUES (${flow}, 'draft', ${sql.json(config as Parameters<typeof sql.json>[0])}, ${est.price * 100}, ${est.deposit * 100},
|
||||
${est.remainder * 100}, ${est.timelineDays}, ${est.estConfidence / 100}, ${key ?? null})
|
||||
RETURNING order_id, created_at`;
|
||||
return {
|
||||
order: {
|
||||
order_id: row.order_id, flow, status: "draft", config,
|
||||
total_ore: est.price * 100, deposit_ore: est.deposit * 100, balance_ore: est.remainder * 100,
|
||||
est_days: est.timelineDays, est_confidence: est.estConfidence / 100, created_at: row.created_at,
|
||||
},
|
||||
replayed: false,
|
||||
};
|
||||
} catch (err) {
|
||||
if (key && isUniqueViolation(err)) {
|
||||
const raced = await findOrderByIdempotencyKey(key);
|
||||
if (raced) {
|
||||
if (sameSubmission(raced, flow, config)) return { order: raced, replayed: true };
|
||||
throw new IdempotencyKeyConflictError(key);
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const id = (globalThis.crypto?.randomUUID?.() ?? `SX-${Math.floor(1000 + Math.random() * 9000)}`);
|
||||
const order: Order = {
|
||||
order_id: id, flow, status: "draft", config,
|
||||
@@ -80,7 +164,8 @@ export async function createOrder(flow: string, config: OrderConfig): Promise<Or
|
||||
est_days: est.timelineDays, est_confidence: est.estConfidence / 100, created_at: new Date().toISOString(),
|
||||
};
|
||||
mem.set(id, order);
|
||||
return order;
|
||||
if (key) memIdempotencyIndex.set(key, id);
|
||||
return { order, replayed: false };
|
||||
}
|
||||
|
||||
export async function getOrder(id: string): Promise<Order | null> {
|
||||
|
||||
+52
-2
@@ -19,10 +19,29 @@
|
||||
*
|
||||
* Health/metrics are exempt (they are also outside the matcher below, but the
|
||||
* exemption is kept explicit for robustness).
|
||||
*
|
||||
* FIX (2026-08-31): matcher now covers BOTH /api/0pt (exact) and /api/0pt/:path*
|
||||
* to close the gap where a bare GET /api/0pt (no trailing path) was routed around
|
||||
* the middleware on Vercel's Edge runtime.
|
||||
*
|
||||
* FIX (P014): one route under this matcher, /api/0pt/quality/bridge-run, is
|
||||
* not called by a customer's Zitadel-authenticated session at all -- it is
|
||||
* called BY the Python engine (services/api/workers/quality_bridge.py) using
|
||||
* the pre-existing X-Internal-Token shared-secret convention, never a Bearer
|
||||
* token. Before this fix, that meant middleware always rejected the engine's
|
||||
* real call with 401 (see src/auth/internal-token.ts's doc comment for the
|
||||
* full story) -- the quality bridge was permanently unreachable in any
|
||||
* environment enforcing auth. INTERNAL_ONLY_ROUTES lets a request to that
|
||||
* ONE path through on a verified X-Internal-Token instead of a Bearer token;
|
||||
* every other /api/0pt/* route is untouched and still requires the normal
|
||||
* Zitadel flow below. A request to that route WITHOUT a valid internal token
|
||||
* still falls through to the ordinary Bearer check (e.g. an operator testing
|
||||
* it manually with their own token), it is not a blanket bypass.
|
||||
*/
|
||||
|
||||
import { NextResponse, type NextRequest } from 'next/server';
|
||||
import { assertAuthConfigSafe, resolveAuthContext } from '@/src/auth/context';
|
||||
import { verifyInternalServiceToken } from '@/src/auth/internal-token';
|
||||
|
||||
// STARTUP GATE: fail to boot rather than serve /api/0pt/* openly in production.
|
||||
assertAuthConfigSafe();
|
||||
@@ -30,10 +49,21 @@ assertAuthConfigSafe();
|
||||
// Only guard the 0pt capability surface. The matcher keeps this middleware off
|
||||
// the UI, auth, and payment routes entirely.
|
||||
export const config = {
|
||||
matcher: ['/api/0pt/:path*'],
|
||||
matcher: ['/api/0pt', '/api/0pt/:path*'],
|
||||
};
|
||||
|
||||
const EXEMPT_PATTERNS: RegExp[] = [/^\/api\/health(\/|$)/, /^\/api\/admin\/metrics(\/|$)/];
|
||||
const EXEMPT_PATTERNS: RegExp[] = [
|
||||
/^\/api\/health(\/|$)/,
|
||||
/^\/api\/ready(\/|$)/,
|
||||
/^\/api\/admin\/metrics(\/|$)/,
|
||||
];
|
||||
|
||||
/** The one 0pt route reached by a service (the Python engine), not a user session. */
|
||||
const INTERNAL_ONLY_ROUTES: RegExp[] = [/^\/api\/0pt\/quality\/bridge-run$/];
|
||||
|
||||
// Module-level diagnostics counter — first 5 auth requests in a process lifetime
|
||||
// log their outcome so the production auth gap (S1) can be traced to root cause.
|
||||
let _authDiagCount = 0;
|
||||
|
||||
export async function middleware(req: NextRequest): Promise<NextResponse> {
|
||||
const path = req.nextUrl.pathname;
|
||||
@@ -41,6 +71,26 @@ export async function middleware(req: NextRequest): Promise<NextResponse> {
|
||||
return NextResponse.next();
|
||||
}
|
||||
|
||||
if (INTERNAL_ONLY_ROUTES.some((re) => re.test(path)) && verifyInternalServiceToken(req.headers)) {
|
||||
// Machine-to-machine call, verified via the shared X-Internal-Token
|
||||
// secret rather than a user's Bearer token. Deliberately does NOT set
|
||||
// x-0pt-verified-* headers -- there is no per-user principal here, and
|
||||
// the route requires+validates tenant_id from its own request body
|
||||
// (qualityBridgeRunSchema), which is the actual tenant the engine is
|
||||
// running the order for.
|
||||
return NextResponse.next();
|
||||
}
|
||||
|
||||
if (_authDiagCount < 5) {
|
||||
_authDiagCount += 1;
|
||||
const hasAuth = !!req.headers.get('authorization');
|
||||
console.log(
|
||||
`[0pt-auth] middleware path=${path} auth_header=${hasAuth} ` +
|
||||
`auth_enforce=${process.env.AUTH_ENFORCE ?? 'unset'} ` +
|
||||
`node_env=${process.env.NODE_ENV ?? 'unset'} request#${_authDiagCount}`,
|
||||
);
|
||||
}
|
||||
|
||||
const decision = await resolveAuthContext(req.headers);
|
||||
|
||||
if (!decision.ok) {
|
||||
|
||||
+35
-32
@@ -14,39 +14,41 @@
|
||||
"test:watch": "vitest",
|
||||
"format": "prettier --check .",
|
||||
"format:write": "prettier --write .",
|
||||
"typecheck:app": "tsc --noEmit -p tsconfig.json"
|
||||
"typecheck:app": "tsc --noEmit -p tsconfig.json",
|
||||
"siax:doctor": "node scripts/siax-doctor.mjs",
|
||||
"deps:check": "depcruise src app components lib --config .dependency-cruiser.cjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@fontsource-variable/inter": "^5.2.5",
|
||||
"@fontsource-variable/jetbrains-mono": "^5.2.5",
|
||||
"@hookform/resolvers": "^3.10.0",
|
||||
"@radix-ui/react-accordion": "1.2.2",
|
||||
"@radix-ui/react-alert-dialog": "1.1.4",
|
||||
"@radix-ui/react-aspect-ratio": "1.1.1",
|
||||
"@radix-ui/react-avatar": "1.1.2",
|
||||
"@radix-ui/react-checkbox": "1.1.3",
|
||||
"@radix-ui/react-collapsible": "1.1.2",
|
||||
"@radix-ui/react-context-menu": "2.2.4",
|
||||
"@radix-ui/react-dialog": "1.1.4",
|
||||
"@radix-ui/react-dropdown-menu": "2.1.4",
|
||||
"@radix-ui/react-hover-card": "1.1.4",
|
||||
"@radix-ui/react-label": "2.1.1",
|
||||
"@radix-ui/react-menubar": "1.1.4",
|
||||
"@radix-ui/react-navigation-menu": "1.2.3",
|
||||
"@radix-ui/react-popover": "1.1.4",
|
||||
"@radix-ui/react-progress": "1.1.1",
|
||||
"@radix-ui/react-radio-group": "1.2.2",
|
||||
"@radix-ui/react-scroll-area": "1.2.2",
|
||||
"@radix-ui/react-select": "2.1.4",
|
||||
"@radix-ui/react-separator": "1.1.1",
|
||||
"@radix-ui/react-slider": "1.2.2",
|
||||
"@radix-ui/react-slot": "1.1.1",
|
||||
"@radix-ui/react-switch": "1.1.2",
|
||||
"@radix-ui/react-tabs": "1.1.2",
|
||||
"@radix-ui/react-toast": "1.2.4",
|
||||
"@radix-ui/react-toggle": "1.1.1",
|
||||
"@radix-ui/react-toggle-group": "1.1.1",
|
||||
"@radix-ui/react-tooltip": "1.1.6",
|
||||
"@radix-ui/react-accordion": "1.2.20",
|
||||
"@radix-ui/react-alert-dialog": "1.1.23",
|
||||
"@radix-ui/react-aspect-ratio": "1.1.15",
|
||||
"@radix-ui/react-avatar": "1.2.6",
|
||||
"@radix-ui/react-checkbox": "1.3.11",
|
||||
"@radix-ui/react-collapsible": "1.1.20",
|
||||
"@radix-ui/react-context-menu": "2.3.7",
|
||||
"@radix-ui/react-dialog": "1.1.23",
|
||||
"@radix-ui/react-dropdown-menu": "2.1.24",
|
||||
"@radix-ui/react-hover-card": "1.1.23",
|
||||
"@radix-ui/react-label": "2.1.15",
|
||||
"@radix-ui/react-menubar": "1.1.24",
|
||||
"@radix-ui/react-navigation-menu": "1.2.22",
|
||||
"@radix-ui/react-popover": "1.1.23",
|
||||
"@radix-ui/react-progress": "1.1.16",
|
||||
"@radix-ui/react-radio-group": "1.4.7",
|
||||
"@radix-ui/react-scroll-area": "1.2.18",
|
||||
"@radix-ui/react-select": "2.3.7",
|
||||
"@radix-ui/react-separator": "1.1.15",
|
||||
"@radix-ui/react-slider": "1.4.7",
|
||||
"@radix-ui/react-slot": "1.3.3",
|
||||
"@radix-ui/react-switch": "1.3.7",
|
||||
"@radix-ui/react-tabs": "1.1.21",
|
||||
"@radix-ui/react-toast": "1.2.23",
|
||||
"@radix-ui/react-toggle": "1.1.18",
|
||||
"@radix-ui/react-toggle-group": "1.1.19",
|
||||
"@radix-ui/react-tooltip": "1.2.16",
|
||||
"autoprefixer": "^10.4.20",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
@@ -56,12 +58,12 @@
|
||||
"input-otp": "1.4.1",
|
||||
"jose": "^5.9.6",
|
||||
"lucide-react": "^0.454.0",
|
||||
"next": "16.0.10",
|
||||
"next": "16.3.3",
|
||||
"next-themes": "^0.4.6",
|
||||
"postgres": "^3.4.9",
|
||||
"react": "19.2.0",
|
||||
"react": "19.2.8",
|
||||
"react-day-picker": "9.8.0",
|
||||
"react-dom": "19.2.0",
|
||||
"react-dom": "19.2.8",
|
||||
"react-hook-form": "^7.60.0",
|
||||
"react-resizable-panels": "^2.1.7",
|
||||
"recharts": "2.15.4",
|
||||
@@ -78,11 +80,12 @@
|
||||
"@types/node": "^22",
|
||||
"@types/react": "^19",
|
||||
"@types/react-dom": "^19",
|
||||
"dependency-cruiser": "^18.2.0",
|
||||
"eslint": "^9.15.0",
|
||||
"postcss": "^8.5",
|
||||
"prettier": "^3.9.6",
|
||||
"tailwindcss": "^4.1.9",
|
||||
"tw-animate-css": "1.3.3",
|
||||
"tw-animate-css": "1.4.0",
|
||||
"typescript": "^5",
|
||||
"typescript-eslint": "^8.15.0",
|
||||
"vitest": "^3.2.7"
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"name": "@siax/0pt-schemas",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"description": "BACKLOG P1-21: makes packages/schemas a real workspace package (it was previously loose JSON Schema files with no package.json, so pnpm-workspace.yaml's `packages/*` glob matched an empty package). Zero build step -- these files are consumed by path, not imported as JS.",
|
||||
"files": ["*.schema.json"],
|
||||
"license": "UNLICENSED"
|
||||
}
|
||||
Generated
+1351
-1063
File diff suppressed because it is too large
Load Diff
@@ -1,7 +1,13 @@
|
||||
# Prompts — index
|
||||
|
||||
**Kontroll-ID:** AGENT-02 (denna katalog + index), AGENT-03 (varje enskild
|
||||
prompt nedan har mål/kontext/förutsättningar/omfattning/kontrakt/
|
||||
acceptanskriterier/kontroll-ID/förbjudna åtgärder)
|
||||
|
||||
> Operativa prompt-mallar för agentsessioner mot 0pt. Konvention: `NNN-roll.md`.
|
||||
> Alla mallar respekterar CLAUDE.md/AGENTS.md + docs/compliance (AR-5).
|
||||
> Självdiagnos: `pnpm siax:doctor` (AGENT-04, `scripts/siax-doctor.mjs`)
|
||||
> kör den maskinellt mätbara delmängden av dessa krav lokalt.
|
||||
|
||||
| Fil | Användning |
|
||||
|---|---|
|
||||
|
||||
@@ -1,22 +1,65 @@
|
||||
# Prompt: Repo-hälsa (0pt)
|
||||
|
||||
**Kontroll-ID:** AGENT-03 (denna mall), AGENT-02 (`prompts/`-strukturen som helhet)
|
||||
|
||||
> Använd före annat arbete: verifiera att utgångsläget är sant innan ändringar.
|
||||
|
||||
## Mål
|
||||
|
||||
Fastställa repots verkliga läge (branch, CI, live `/api/health`) innan någon
|
||||
kod ändras, så att arbetet bygger på fakta, inte på vad ett dokument påstår.
|
||||
|
||||
## Kontext
|
||||
|
||||
`main` på detta repo har varit mål för flera parallella sessioner (se
|
||||
`docs/compliance/BACKLOG.md`s löpande historik). Antaganden om "senaste
|
||||
läget" blir fel snabbt — kör alltid denna prompt först.
|
||||
|
||||
## Förutsättningar
|
||||
|
||||
- Läs-åtkomst till `git.siax.io/sax3l/0pt` (klon eller `git fetch`).
|
||||
- Nätåtkomst till `https://0pt.app` för live-proben (annars: notera EJ
|
||||
VERIFIERAT i output, gissa aldrig).
|
||||
|
||||
## Omfattning
|
||||
|
||||
Läsning + rapportering endast. Denna prompt gör inga kodändringar.
|
||||
|
||||
## 1. Status
|
||||
|
||||
- `git status` + `git log --oneline -5` — ren main?
|
||||
- Senaste CI-run (Gitea Actions API):
|
||||
`/repos/sax3l/0pt/actions/runs?limit=5` — ci.yml ska vara grön;
|
||||
cd.yml/deploy.yml röd = avsiktlig guard (placeholder UUID).
|
||||
- `curl -s https://0pt.app/api/health` — ok/true + db/queue-läge.
|
||||
`/repos/sax3l/0pt/actions/runs?limit=5` — `ci.yml` ska vara grön;
|
||||
`cd.yml` röd = avsiktlig guard (placeholder UUID).
|
||||
- `curl -s https://0pt.app/api/health` — `ok`/beroendestatus (se BACKLOG P0-3:
|
||||
fältet rapporterar numera riktig per-dependency-status, inte bara
|
||||
`DATABASE_URL`/`REDIS_URL`-närvaro).
|
||||
|
||||
## 2. Luckor att minnas
|
||||
|
||||
- S1: `/api/0pt/*` svarade 200 anonymt (2026-08-10) — påstå aldrig "skyddad".
|
||||
- S2: tenant tas ur body — nya handlers läser `x-0pt-verified-tenant`.
|
||||
- Bypass-stacken `services/scrapers/*` — medvetet behållen, rör ej.
|
||||
- S1: `/api/0pt/*` svarade 200 anonymt (2026-08-10) — påstå aldrig "skyddad"
|
||||
utan en färsk 401-verifiering.
|
||||
- S2: tenant tas ur body i äldre handlers — nya handlers läser
|
||||
`x-0pt-verified-tenant`.
|
||||
- Bypass-stacken `services/scrapers/*` — medvetet behållen, rör ej (se
|
||||
`SECURITY.md`).
|
||||
|
||||
## Kontrakt / maskinellt verifierbara acceptanskriterier
|
||||
|
||||
- [ ] `git log -1 --format=%H` rapporterad i output.
|
||||
- [ ] CI-status för senaste `ci.yml`-run på `main` rapporterad (grön/röd + run-ID).
|
||||
- [ ] `/api/health`-svarets `ok`-fält + per-dependency-status citerat ordagrant
|
||||
(inte omskrivet) med tidsstämpel för när proben kördes.
|
||||
- [ ] Status på S1–S3 märkt VERIFIERAT/ANTAGANDE/EJ VERIFIERAT — aldrig
|
||||
olabellat.
|
||||
|
||||
## 3. Output
|
||||
|
||||
Rapportera: HEAD-SHA, CI-status, /api/health-svar, status på S1–S3
|
||||
Rapportera: HEAD-SHA, CI-status, `/api/health`-svar, status på S1–S3
|
||||
(VERIFIERAT/ANTAGANDE/EJ VERIFIERAT).
|
||||
|
||||
## Förbjudna åtgärder
|
||||
|
||||
- Skriv aldrig till `main` från denna prompt.
|
||||
- Påstå aldrig ett kontrollresultat utan att ha kört den bakomliggande
|
||||
kommandot/proben denna session.
|
||||
|
||||
@@ -1,7 +1,33 @@
|
||||
# Prompt: Compliance-audit (0pt)
|
||||
|
||||
**Kontroll-ID:** AGENT-03 (denna mall), AGENT-04/AGENT-05 (denna prompt är den
|
||||
manuella/agent-drivna motsvarigheten till `pnpm siax:doctor` + framtida
|
||||
C0NF0RM-rapportering — se `scripts/siax-doctor.mjs`)
|
||||
|
||||
> Uppföljning av GOLDEN-STANDARD-REPORT. Körs när backlog-objekt stängts.
|
||||
|
||||
## Mål
|
||||
|
||||
Hålla `docs/compliance/GOLDEN-STANDARD-REPORT.md` och `docs/compliance/BACKLOG.md`
|
||||
i synk med repots faktiska läge — aldrig med önskeläget.
|
||||
|
||||
## Kontext
|
||||
|
||||
Poängen i GOLDEN-STANDARD-REPORT.md är avsedd att jämföras mot andra SIAX
|
||||
Cloud-repon. En falskt hög poäng (ändrad utan nytt bevis) gör den jämförelsen
|
||||
värdelös för hela flottan, inte bara för detta repo.
|
||||
|
||||
## Förutsättningar
|
||||
|
||||
- `docs/compliance/GOLDEN-STANDARD-REPORT.md` och `docs/compliance/BACKLOG.md`
|
||||
existerar och lästa i sin helhet innan ändring.
|
||||
- Det påstådda beviset (testfil, commit-SHA, live-svar) finns och går att peka på.
|
||||
|
||||
## Omfattning
|
||||
|
||||
Endast `docs/compliance/**`. Kör aldrig kodändringar från denna prompt — de
|
||||
hör hemma i den ändring som faktiskt stänger backlog-posten.
|
||||
|
||||
## 1. Uppdatera
|
||||
|
||||
- Läs `docs/compliance/GOLDEN-STANDARD-REPORT.md` — sätt status ✅/⚠️/❌/➖
|
||||
@@ -17,3 +43,17 @@
|
||||
attribuerade beslut (Simon Axelsson, 2026-08-11, se SECURITY.md), INTE det
|
||||
gamla oattribuerade "medvetet användarbeslut" — ingen ändring av C6-status.
|
||||
- Inga fabricerade förbättringar: om inget faktiskt ändrats → ingen poängändring.
|
||||
|
||||
## Kontrakt / maskinellt verifierbara acceptanskriterier
|
||||
|
||||
- [ ] Varje ✅/⚠️→bättre statusändring har en bevisrad (filväg/SHA/kommando)
|
||||
i samma commit.
|
||||
- [ ] Poängräkningen i "## Poäng" summerar korrekt om man räknar om den
|
||||
(✅×1 + ⚠️×0.5) / (antal kontroller − antal ➖).
|
||||
- [ ] C6 oförändrad ➖ om inte en ny, daterad, attribuerad ägarinstruktion citeras.
|
||||
|
||||
## Förbjudna åtgärder
|
||||
|
||||
- Ändra aldrig C6:s status eller motivering utan en ny, explicit,
|
||||
daterad instruktion från kontoägaren i denna session.
|
||||
- Höj aldrig en kontrolls poäng utan ett konkret, citerat bevis.
|
||||
|
||||
@@ -1,19 +1,63 @@
|
||||
# Prompt: Security review (0pt)
|
||||
|
||||
**Kontroll-ID:** AGENT-03 (denna mall)
|
||||
|
||||
> Säkerhetsgranskning av en ändring före push. Kör ALLTID vid ändringar i
|
||||
> auth, klienter (cl0ud/aud0/n0d/st0re), payments eller middleware.
|
||||
|
||||
## Mål
|
||||
|
||||
Fånga regressions i auth/tenant/hemlighetshantering innan de når `main`,
|
||||
komplement till (inte ersättning för) CI:s blockerande Trivy/gitleaks-steg
|
||||
(`.gitea/workflows/security.yml`).
|
||||
|
||||
## Kontext
|
||||
|
||||
Två av repots historiskt allvarligaste fynd (S1 anonym `/api/0pt/*`, S3
|
||||
anonym `/api/orders`) hade upptäckts av denna checklista om den körts före
|
||||
push. CI:s automatiska scanning (BACKLOG P0-16/P0-19) täcker beroenden och
|
||||
hemligheter i historiken — den täcker inte logikfel som "tenant_id läst ur
|
||||
body".
|
||||
|
||||
## Förutsättningar
|
||||
|
||||
- Diffen som ska granskas är känd (branch/PR-nummer).
|
||||
- `pnpm run typecheck` och `pnpm run test` går att köra lokalt.
|
||||
|
||||
## Omfattning
|
||||
|
||||
Granskning av EN ändring/PR. Ändrar inget i sig själv — output är en rapport,
|
||||
inte en fix.
|
||||
|
||||
## 1. Checklista
|
||||
|
||||
- [ ] Inga hemligheter i diffen (grep: sk_/whsec/private key/AIza/AKIA…).
|
||||
- [ ] Inga hemligheter i diffen (grep: sk_/whsec/private key/AIza/AKIA…) —
|
||||
CI:s gitleaks-steg är facit, men kör lokalt först.
|
||||
- [ ] Inga nya betald-SaaS-/Google/Vercel-bärare (AR-4).
|
||||
- [ ] `tenant_id` inte tagen ur request-body (S2-regeln).
|
||||
- [ ] Inga ändringar i `services/scrapers/*` (bypass-stacken, medvetet beslut).
|
||||
- [ ] Auth-fail-closed kvarstår; ingen `AUTH_ENFORCE=false`-produktionsväg.
|
||||
- [ ] Inga `console.log` av hemligheter; ingen `NEXT_PUBLIC_*`-hemlighet.
|
||||
- [ ] Tester täcker nya grenar; typecheck+test gröna.
|
||||
- [ ] Trivy-fynd i diffen triagerade och dokumenterade.
|
||||
- [ ] Trivy-fynd i diffen triagerade och dokumenterade (BACKLOG P0-16: Trivy
|
||||
är numera blockerande i CI — ett rött `trivy`-jobb är förväntat tills
|
||||
befintliga beroende-fynd åtgärdas separat, inte en regression du orsakat
|
||||
om det redan var rött på `main`).
|
||||
|
||||
## Kontrakt / maskinellt verifierbara acceptanskriterier
|
||||
|
||||
- [ ] Varje checklisterad punkt har VERIFIERAT/ANTAGANDE/EJ VERIFIERAT +
|
||||
risknivå (L0–L5) i output — ingen punkt lämnas tom.
|
||||
- [ ] Om `services/scrapers/*` finns i diffen: granskningen stoppar och
|
||||
eskalerar, gör ingen egen bedömning.
|
||||
|
||||
## 2. Output
|
||||
|
||||
`VERIFIERAT/ANTAGANDE/EJ VERIFIERAT` per punkt + risknivå (L0–L5) + rekommendation.
|
||||
|
||||
## Förbjudna åtgärder
|
||||
|
||||
- Godkänn aldrig en diff som rör `services/scrapers/*`,
|
||||
`services/infra/docker-compose.data-stack.yml` eller
|
||||
`live/src/workers/adaptive_router.py`s `curl_cffi`-rader — eskalera i stället.
|
||||
- Fatta aldrig auktoritativt beslut om C6/bypass-stacken från denna prompt.
|
||||
|
||||
@@ -1,8 +1,35 @@
|
||||
# Prompt: Deploy-cutover (0pt)
|
||||
|
||||
**Kontroll-ID:** AGENT-03 (denna mall)
|
||||
|
||||
> Mål: 0pt.app på Coolify, sedan Vercel-avveckling. Följ
|
||||
> `docs/runbook/deploy.md` steg för steg.
|
||||
|
||||
## Mål
|
||||
|
||||
Flytta produktionstrafik från Vercel till den provisionerade Coolify-appen
|
||||
utan dataförlust eller nedtid, och utan att avveckla Vercel förrän paritet
|
||||
är bevisad.
|
||||
|
||||
## Kontext
|
||||
|
||||
`siax.config.json` deklarerar redan `deploy.target: "coolify"` (ADR-0001)
|
||||
— detta är avsiktligt bara en konfigurationsavsikt, inte en genomförd
|
||||
flytt. Ingen Coolify-app finns provisionerad för detta repo än
|
||||
(`.gitea/workflows/cd.yml`s `COOLIFY_APP_ID` är en medveten platshållare).
|
||||
|
||||
## Förutsättningar
|
||||
|
||||
- Coolify-åtkomst (cloud.siax.io) för att skapa appen.
|
||||
- Infisical-projekt för 0pt (BACKLOG P0-1 — **kräver en Infisical-token som
|
||||
denna prompts session inte nödvändigtvis har**; verifiera innan steg 1).
|
||||
- `COOLIFY_TOKEN`-repo-secret-behörighet.
|
||||
|
||||
## Omfattning
|
||||
|
||||
Infra-provisionering + DNS + en skyddad-sökväg-PR (`cd.yml`). Rör aldrig
|
||||
`services/scrapers/*` eller andra frysta sökvägar under denna cutover.
|
||||
|
||||
## 1. Gate (blockerare)
|
||||
|
||||
- Coolify-app existerar? (cloud.siax.io → projekt)
|
||||
@@ -15,10 +42,28 @@
|
||||
1. Provisionera app (dockerfile, repo sax3l/0pt, branch main).
|
||||
2. PR: fyll UUID + verifiera status-check (skyddade filer).
|
||||
3. Kör deploy, följ Gitea-run → grön.
|
||||
4. `curl https://0pt.app/api/health` och en marknadssida.
|
||||
4. `curl https://0pt.app/api/health` **och** `curl https://0pt.app/api/ready`
|
||||
(BACKLOG P0-3 — readiness failar stängt om databasen inte är uppe; kräv
|
||||
200 på båda, inte bara `/api/health`) och en marknadssida.
|
||||
5. DNS-ompekning + Vercel-avveckling först när paritet certifierats.
|
||||
|
||||
## 3. Regel
|
||||
|
||||
Radera aldrig Vercel-projektet före certifierad paritet. Dokumentera varje steg
|
||||
i `docs/compliance/EVIDENCE/` med datum + SHA.
|
||||
|
||||
## Kontrakt / maskinellt verifierbara acceptanskriterier
|
||||
|
||||
- [ ] `/api/health` OCH `/api/ready` svarar 200 på den nya Coolify-URL:en
|
||||
innan DNS pekas om.
|
||||
- [ ] Varje steg i sekvensen har en motsvarande rad i
|
||||
`docs/compliance/EVIDENCE/` med datum + SHA innan nästa steg påbörjas.
|
||||
- [ ] Vercel-projektet finns kvar (overifierat raderat) tills steg 5 är klart.
|
||||
|
||||
## Förbjudna åtgärder
|
||||
|
||||
- Radera eller pausa Vercel-projektet före certifierad paritet.
|
||||
- Fyll `COOLIFY_APP_ID` eller `COOLIFY_TOKEN` med gissade/platshållarvärden
|
||||
som INTE är den verkliga provisionerade appens uppgifter.
|
||||
- Rör `services/scrapers/*`, `services/infra/docker-compose.data-stack.yml`
|
||||
under denna cutover — helt orelaterat till deploy-målet.
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env node
|
||||
// siax-doctor (BACKLOG P1-19, control AGENT-04): "Repot är
|
||||
// self-diagnostiserande: `pnpm siax:doctor` kör hela standarden lokalt och
|
||||
// skriver ut avvikelser med kontroll-ID."
|
||||
//
|
||||
// This is a repo-local, zero-dependency subset -- NOT the published
|
||||
// @siax/doctor package from sax3l/siax-standard. That package was
|
||||
// deliberately not taken on as a dependency here: at the time this was
|
||||
// written it wasn't verified reachable/stable from this repo's frozen
|
||||
// lockfile install, and a doctor script that can silently fail to install
|
||||
// is worse than no doctor script. This checks the same class of
|
||||
// machine-checkable facts (required files, manifest shape) using only
|
||||
// Node's built-in `fs`/`path` -- no network, no registry.
|
||||
//
|
||||
// Exit code 0 = every check passed. Exit code 1 = at least one check failed
|
||||
// (see the printed control-ID to find the BACKLOG/standard item).
|
||||
//
|
||||
// NOTE for reviewers: this script was authored alongside two sibling PRs
|
||||
// (fix/honest-health-check adds app/api/ready/route.ts;
|
||||
// ci/security-headers-and-scanning adds CODEOWNERS) that this branch does
|
||||
// not itself contain. Run this on a checkout with all three merged for a
|
||||
// clean 0-failure result -- a couple of FAILs on this branch alone, for
|
||||
// exactly those two files, is expected and not a bug in either PR.
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
|
||||
/** @type {{id: string, label: string, check: () => boolean | string}[]} */
|
||||
const checks = [];
|
||||
|
||||
function file(path) {
|
||||
return join(ROOT, path);
|
||||
}
|
||||
|
||||
function requireFile(id, label, path) {
|
||||
checks.push({
|
||||
id,
|
||||
label,
|
||||
check: () => (existsSync(file(path)) ? true : `missing: ${path}`),
|
||||
});
|
||||
}
|
||||
|
||||
function requireJsonField(id, label, path, fieldPath) {
|
||||
checks.push({
|
||||
id,
|
||||
label,
|
||||
check: () => {
|
||||
if (!existsSync(file(path))) return `missing file: ${path}`;
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse(readFileSync(file(path), "utf8"));
|
||||
} catch (e) {
|
||||
return `invalid JSON in ${path}: ${e.message}`;
|
||||
}
|
||||
let cur = data;
|
||||
for (const key of fieldPath.split(".")) {
|
||||
if (cur == null || !(key in cur)) return `missing field "${fieldPath}" in ${path}`;
|
||||
cur = cur[key];
|
||||
}
|
||||
return true;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function requireTextMatch(id, label, path, pattern, patternLabel) {
|
||||
checks.push({
|
||||
id,
|
||||
label,
|
||||
check: () => {
|
||||
if (!existsSync(file(path))) return `missing file: ${path}`;
|
||||
const text = readFileSync(file(path), "utf8");
|
||||
return pattern.test(text) ? true : `${path} does not contain ${patternLabel}`;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// -- REPO-* / DOC-* : required root files --------------------------------
|
||||
requireFile("DOC-01", "README.md exists", "README.md");
|
||||
requireFile("REPO-08", "LICENSE exists", "LICENSE");
|
||||
requireFile("REPO-13", ".editorconfig exists", ".editorconfig");
|
||||
checks.push({
|
||||
id: "REPO-15/P0-19",
|
||||
label: "no committed .env (real secrets)",
|
||||
check: () => (existsSync(file(".env")) ? "a real .env file is committed -- should never happen" : true),
|
||||
});
|
||||
requireFile("P1-15", "CODEOWNERS exists", "CODEOWNERS");
|
||||
requireFile("P1-3", "CHANGELOG.md exists", "CHANGELOG.md");
|
||||
requireFile("SEC-*", "SECURITY.md exists", "SECURITY.md");
|
||||
requireFile("AGENT-01", "AGENTS.md exists", "AGENTS.md");
|
||||
requireFile("DOC-01", "CONTRIBUTING.md exists", "CONTRIBUTING.md");
|
||||
|
||||
// -- DOC-02/DOC-04 : ARCHITECTURE.md at the exact standard path -----------
|
||||
requireFile("DOC-02", "docs/architecture/ARCHITECTURE.md exists at the standard path", "docs/architecture/ARCHITECTURE.md");
|
||||
requireTextMatch(
|
||||
"DOC-02/v2.0",
|
||||
"ARCHITECTURE.md has the v2.0 '## 5. Ytor och paritet' section",
|
||||
"docs/architecture/ARCHITECTURE.md",
|
||||
/^## 5\. Ytor och paritet/m,
|
||||
"'## 5. Ytor och paritet'",
|
||||
);
|
||||
requireTextMatch(
|
||||
"DOC-04",
|
||||
"ARCHITECTURE.md has a 'Senast verifierad mot kod' date field",
|
||||
"docs/architecture/ARCHITECTURE.md",
|
||||
/Senast verifierad mot kod:\*\*\s*\d{4}-\d{2}-\d{2}/,
|
||||
"a dated 'Senast verifierad mot kod' field",
|
||||
);
|
||||
|
||||
// -- P1-1 / REPO-03/04/06 : manifests --------------------------------------
|
||||
requireFile("P1-1/REPO-04", "siax.repo.json exists", "siax.repo.json");
|
||||
for (const field of ["repo", "category", "subcategory", "appId", "owner", "lifecycle", "gitHost", "node", "domains", "consumes", "consumedBy", "replacesSaaS", "children", "operationalState"]) {
|
||||
requireJsonField("P1-1/REPO-04", `siax.repo.json has required field "${field}"`, "siax.repo.json", field);
|
||||
}
|
||||
requireFile("P1-1/REPO-06", "siax.config.json exists", "siax.config.json");
|
||||
for (const field of ["framework", "build", "start", "port", "healthcheck", "environments"]) {
|
||||
requireJsonField("P1-1/REPO-06", `siax.config.json has required field "${field}"`, "siax.config.json", field);
|
||||
}
|
||||
|
||||
// -- AGENT-02/03 : prompts -------------------------------------------------
|
||||
requireFile("AGENT-02", "prompts/000-INDEX.md exists", "prompts/000-INDEX.md");
|
||||
|
||||
// -- DX-03 : devcontainer --------------------------------------------------
|
||||
requireFile("DX-03", ".devcontainer/devcontainer.json exists", ".devcontainer/devcontainer.json");
|
||||
|
||||
// -- P1-2 : ADRs -------------------------------------------------------------
|
||||
requireFile("P1-2", "docs/architecture/decisions/ has at least one ADR", "docs/architecture/decisions/README.md");
|
||||
|
||||
// -- CI-14 : contract ---------------------------------------------------------
|
||||
requireFile("A4/DOC", "contracts/openapi.yaml exists", "contracts/openapi.yaml");
|
||||
|
||||
// -- CD-04/OBS-04 : health/ready ---------------------------------------------
|
||||
requireFile("P0-3/CD-04", "app/api/health/route.ts exists", "app/api/health/route.ts");
|
||||
requireFile("P0-3/OBS-04", "app/api/ready/route.ts exists (readiness, not just liveness)", "app/api/ready/route.ts");
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
let failures = 0;
|
||||
console.log("siax-doctor -- 0pt (repo-local subset, see scripts/siax-doctor.mjs header)\n");
|
||||
for (const { id, label, check } of checks) {
|
||||
const result = check();
|
||||
if (result === true) {
|
||||
console.log(` [PASS] ${id.padEnd(14)} ${label}`);
|
||||
} else {
|
||||
failures += 1;
|
||||
console.log(` [FAIL] ${id.padEnd(14)} ${label} -- ${result}`);
|
||||
}
|
||||
}
|
||||
console.log(`\n${checks.length - failures}/${checks.length} checks passed.`);
|
||||
if (failures > 0) {
|
||||
console.log(`${failures} check(s) failed -- see control-IDs above and cross-reference docs/compliance/BACKLOG.md.`);
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -6,6 +6,7 @@ is the customer-facing edge; this service runs the actual pipeline.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
@@ -76,6 +77,21 @@ def compute_estimate(b: EstimateIn) -> dict[str, Any]:
|
||||
# ── in-memory store (swapped for asyncpg when DATABASE_URL is set) ───────────
|
||||
_ORDERS: dict[str, dict[str, Any]] = {}
|
||||
_JOBS: dict[str, dict[str, Any]] = {}
|
||||
# P014: one asyncio.Lock per order_id, held for the duration of a /run call.
|
||||
# Process-local only (matches the honest in-memory-store limitation this file
|
||||
# already documents), but closes the actual race a bare `if oid in _JOBS`
|
||||
# check would leave open: two /run requests for the same order_id arriving
|
||||
# back to back both await run_job() before either has stored a result, so a
|
||||
# read-then-write check alone would let both through. See `run()` below.
|
||||
_JOB_LOCKS: dict[str, asyncio.Lock] = {}
|
||||
|
||||
|
||||
def _lock_for(oid: str) -> asyncio.Lock:
|
||||
lock = _JOB_LOCKS.get(oid)
|
||||
if lock is None:
|
||||
lock = asyncio.Lock()
|
||||
_JOB_LOCKS[oid] = lock
|
||||
return lock
|
||||
|
||||
|
||||
class OrderIn(BaseModel):
|
||||
@@ -164,12 +180,31 @@ async def set_status(oid: str, body: StatusIn):
|
||||
|
||||
@app.post("/orders/{oid}/run")
|
||||
async def run(oid: str):
|
||||
"""
|
||||
P014: idempotent delivery. An order should be fulfilled/delivered ONCE.
|
||||
Before this fix, calling this endpoint twice for the same order_id (a
|
||||
retried request, a duplicate trigger, an operator double-click) re-ran
|
||||
the full compliant sourcing pipeline a second time and silently
|
||||
overwrote `_JOBS[oid]` with a second, independently-produced result --
|
||||
a genuinely different delivery (a second live fetch against upstream
|
||||
registries, a second QA pass with its own confidence draws), completely
|
||||
indistinguishable from the first at the API surface. This is the actual
|
||||
"duplicate delivery" gap P014's recon flagged as NOT MET: nothing
|
||||
downstream of order creation was idempotent at all. A job already
|
||||
recorded for this order_id is now returned unchanged; `run_job` is never
|
||||
invoked twice for the same order in this process's lifetime. The lock
|
||||
(see `_lock_for` above) closes the race between two requests arriving
|
||||
concurrently, not just sequential retries.
|
||||
"""
|
||||
o = _ORDERS.get(oid)
|
||||
if not o:
|
||||
raise HTTPException(404, "order saknas")
|
||||
o["status"] = "in_production"
|
||||
_JOBS[oid] = await run_job(oid, o["config"])
|
||||
o["status"] = "review"
|
||||
async with _lock_for(oid):
|
||||
if oid in _JOBS:
|
||||
return _JOBS[oid]
|
||||
o["status"] = "in_production"
|
||||
_JOBS[oid] = await run_job(oid, o["config"])
|
||||
o["status"] = "review"
|
||||
return _JOBS[oid]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
[pytest]
|
||||
# services/api's modules import each other as top-level packages
|
||||
# (`from workers.pipeline import run_job`, `from sources.connectors import ...`)
|
||||
# relative to this directory — orchestrator/main.py does the same. Running
|
||||
# pytest from services/api (or with this file as rootdir) needs that on
|
||||
# sys.path; pytest 8's rootdir-relative testpaths + this ini being IN
|
||||
# services/api handles that via pytest's automatic rootdir insertion for
|
||||
# rootdir-relative imports when there is no conftest.py package structure.
|
||||
testpaths = tests
|
||||
asyncio_mode = auto
|
||||
pythonpath = .
|
||||
@@ -0,0 +1,7 @@
|
||||
# Test-only dependencies for services/api. NOT installed in the production
|
||||
# image (Dockerfile only installs requirements.txt) — see pytest.ini for how
|
||||
# the test suite is run locally / in CI.
|
||||
-r requirements.txt
|
||||
pytest==8.3.4
|
||||
pytest-asyncio==0.25.2
|
||||
respx==0.22.0
|
||||
@@ -10,14 +10,23 @@ EXPLICITLY NOT PRESENT (and must never be added here):
|
||||
rotation to evade bans, scraping behind login walls, fake-account creation.
|
||||
|
||||
Every connector returns a normalized company dict + per-field provenance rows
|
||||
(source_id, confidence 0-1, level 0-3, freshness) so nothing ships unsourced.
|
||||
so nothing ships unsourced. `Provenance` (below) carries the full field set
|
||||
BEFORE_STATE.md §3 found missing from the actual dataflow — this repo's
|
||||
5-field bridge_provenance was: source, confidence, level, and a DATE-only
|
||||
freshness; the concept "bridge_provenance" as a named thing did not exist
|
||||
anywhere in the repo at all. This adds, per field: retrieval TIMESTAMP
|
||||
(distinct from the legacy date-only `freshness`), the transform applied,
|
||||
connector/schema version, the producing system, and two slots
|
||||
(`quality`/`entity_resolution`) that downstream stages (quality bridge,
|
||||
resolve.py) fill in as the record moves through the pipeline — they start
|
||||
`None` here and are never backfilled with a guess.
|
||||
Terms change — VERIFY each source's current access/licence before relying.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
@@ -26,6 +35,52 @@ import httpx
|
||||
USER_AGENT = "0pt-data-engine/0.1 (+https://0pt.app; SIAX Technology AB; hej@siax.io)"
|
||||
_TIMEOUT = httpx.Timeout(15.0, connect=10.0)
|
||||
|
||||
# Bumped when a connector's parsing/field-mapping logic changes in a way that
|
||||
# could shift output shape — recorded per-field as `Provenance.connector_version`
|
||||
# so a downstream consumer can tell which parsing logic produced a value.
|
||||
CONNECTOR_VERSIONS: dict[str, str] = {
|
||||
"gleif": "gleif-connector@1",
|
||||
"brreg": "brreg-connector@1",
|
||||
"companies_house": "companies-house-connector@1",
|
||||
"prh": "prh-connector@1",
|
||||
"cvr": "cvr-connector@1",
|
||||
"rdw": "rdw-connector@1",
|
||||
}
|
||||
|
||||
PRODUCER = "0pt-engine"
|
||||
|
||||
# P014: statuses a compliant source uses to signal "stop asking me this way"
|
||||
# (rate-limited, forbidden, or a legal/geo block) rather than "this record
|
||||
# doesn't exist". Collapsing these into an ordinary None (as every connector
|
||||
# did before this pass) is not itself a compliance violation -- nothing here
|
||||
# ever escalates to a bypass -- but it silently discarded a real signal:
|
||||
# adaptive_router.py had no way to know a source had just told it to back
|
||||
# off, so nothing could avoid hammering a rate-limited endpoint on the very
|
||||
# next record. See SourceBlockedError and adaptive_router.record_blocked.
|
||||
BLOCKED_STATUSES = frozenset({401, 403, 429, 451})
|
||||
|
||||
|
||||
class SourceBlockedError(Exception):
|
||||
"""
|
||||
Raised by a connector when the source responds with one of
|
||||
BLOCKED_STATUSES. This is informational, never an instruction to retry
|
||||
harder or escalate -- callers (pipeline.py's _fetch loop) catch it only
|
||||
to record the block via adaptive_router.record_blocked and move on to
|
||||
the next compliant source in the plan, exactly as they already would for
|
||||
a plain None. Compliance stance (CLAUDE.md §1) is unchanged: a block is
|
||||
always a stop signal for THAT source, never a cue to bypass it.
|
||||
"""
|
||||
|
||||
def __init__(self, source_id: str, status_code: int):
|
||||
self.source_id = source_id
|
||||
self.status_code = status_code
|
||||
super().__init__(f"{source_id} signaled a block (HTTP {status_code})")
|
||||
|
||||
|
||||
def _raise_if_blocked(source_id: str, response: httpx.Response) -> None:
|
||||
if response.status_code in BLOCKED_STATUSES:
|
||||
raise SourceBlockedError(source_id, response.status_code)
|
||||
|
||||
|
||||
def _client() -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(
|
||||
@@ -41,7 +96,39 @@ class Provenance:
|
||||
source_id: str
|
||||
confidence: float
|
||||
level: int # 0 unavailable/licensed, 1 single, 2 partial, 3 corroborated
|
||||
freshness: str | None = None
|
||||
freshness: str | None = None # legacy: retrieval DATE only (kept for back-compat)
|
||||
retrieved_at: str | None = None # full ISO-8601 UTC timestamp of the actual fetch
|
||||
transform: str | None = None # e.g. "raw_fetch", "normalized", "entity_resolution_merge"
|
||||
connector_version: str | None = None # e.g. "brreg-connector@1" — see CONNECTOR_VERSIONS
|
||||
producer: str = PRODUCER # which system produced this field's value
|
||||
quality: dict[str, Any] | None = None # filled by the quality bridge, if/when it runs (never guessed)
|
||||
entity_resolution: dict[str, Any] | None = None # filled by resolve.py on merge, never at fetch time
|
||||
artifact_ref: str | None = None # delivery artifact reference (ST0RE), set at delivery time — not yet wired
|
||||
|
||||
|
||||
def make_provenance(
|
||||
field_name: str,
|
||||
source_id: str,
|
||||
confidence: float,
|
||||
level: int,
|
||||
*,
|
||||
transform: str = "raw_fetch",
|
||||
) -> Provenance:
|
||||
"""Factory used by every connector below so the new provenance fields
|
||||
(retrieved_at, connector_version, producer) are populated consistently
|
||||
instead of each call site re-deriving them ad hoc."""
|
||||
now = datetime.now(timezone.utc)
|
||||
return Provenance(
|
||||
field_name=field_name,
|
||||
source_id=source_id,
|
||||
confidence=confidence,
|
||||
level=level,
|
||||
freshness=now.date().isoformat(),
|
||||
retrieved_at=now.isoformat(),
|
||||
transform=transform,
|
||||
connector_version=CONNECTOR_VERSIONS.get(source_id),
|
||||
producer=PRODUCER,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -57,15 +144,12 @@ class CompanyResult:
|
||||
source: str = ""
|
||||
|
||||
|
||||
def _today() -> str:
|
||||
return date.today().isoformat()
|
||||
|
||||
|
||||
# ── GLEIF / LEI (global cross-border key) — open API ────────────────────────
|
||||
async def gleif_by_lei(lei: str) -> CompanyResult | None:
|
||||
url = f"https://api.gleif.org/api/v1/lei-records/{lei}"
|
||||
async with _client() as c:
|
||||
r = await c.get(url, headers={"Accept": "application/vnd.api+json"})
|
||||
_raise_if_blocked("gleif", r)
|
||||
if r.status_code != 200:
|
||||
return None
|
||||
a = (r.json().get("data") or {}).get("attributes") or {}
|
||||
@@ -79,10 +163,9 @@ async def gleif_by_lei(lei: str) -> CompanyResult | None:
|
||||
"country": legal.get("country"), "postal": legal.get("postalCode")},
|
||||
source="gleif",
|
||||
)
|
||||
f = _today()
|
||||
res.provenance += [
|
||||
Provenance("lei", "gleif", 0.99, 3, f),
|
||||
Provenance("name", "gleif", 0.97, 3, f),
|
||||
make_provenance("lei", "gleif", 0.99, 3),
|
||||
make_provenance("name", "gleif", 0.97, 3),
|
||||
]
|
||||
return res
|
||||
|
||||
@@ -92,6 +175,7 @@ async def brreg_by_orgnr(orgnr: str) -> CompanyResult | None:
|
||||
url = f"https://data.brreg.no/enhetsregisteret/api/enheter/{orgnr}"
|
||||
async with _client() as c:
|
||||
r = await c.get(url)
|
||||
_raise_if_blocked("brreg", r)
|
||||
if r.status_code != 200:
|
||||
return None
|
||||
d = r.json()
|
||||
@@ -102,11 +186,10 @@ async def brreg_by_orgnr(orgnr: str) -> CompanyResult | None:
|
||||
address={"raw": d.get("forretningsadresse")},
|
||||
source="brreg",
|
||||
)
|
||||
f = _today()
|
||||
res.provenance += [
|
||||
Provenance("name", "brreg", 0.97, 3, f),
|
||||
Provenance("status", "brreg", 0.95, 3, f),
|
||||
Provenance("address", "brreg", 0.95, 3, f),
|
||||
make_provenance("name", "brreg", 0.97, 3),
|
||||
make_provenance("status", "brreg", 0.95, 3),
|
||||
make_provenance("address", "brreg", 0.95, 3),
|
||||
]
|
||||
return res
|
||||
|
||||
@@ -117,10 +200,11 @@ async def companies_house_by_number(number: str) -> CompanyResult | None:
|
||||
if not key:
|
||||
# No key -> deliver honestly as unavailable, never fabricate.
|
||||
return CompanyResult(org_nr=number, market="GB", source="companies_house",
|
||||
provenance=[Provenance("name", "companies_house", 0.0, 0, _today())])
|
||||
provenance=[make_provenance("name", "companies_house", 0.0, 0, transform="unavailable_no_key")])
|
||||
url = f"https://api.company-information.service.gov.uk/company/{number}"
|
||||
async with _client() as c:
|
||||
r = await c.get(url, auth=(key, ""))
|
||||
_raise_if_blocked("companies_house", r)
|
||||
if r.status_code != 200:
|
||||
return None
|
||||
d = r.json()
|
||||
@@ -131,10 +215,9 @@ async def companies_house_by_number(number: str) -> CompanyResult | None:
|
||||
address={"raw": d.get("registered_office_address")},
|
||||
source="companies_house",
|
||||
)
|
||||
f = _today()
|
||||
res.provenance += [
|
||||
Provenance("name", "companies_house", 0.97, 3, f),
|
||||
Provenance("status", "companies_house", 0.95, 3, f),
|
||||
make_provenance("name", "companies_house", 0.97, 3),
|
||||
make_provenance("status", "companies_house", 0.95, 3),
|
||||
]
|
||||
return res
|
||||
|
||||
@@ -144,6 +227,7 @@ async def prh_by_businessid(business_id: str) -> CompanyResult | None:
|
||||
url = f"https://avoindata.prh.fi/opendata-ytj-api/v3/companies?businessId={business_id}"
|
||||
async with _client() as c:
|
||||
r = await c.get(url)
|
||||
_raise_if_blocked("prh", r)
|
||||
if r.status_code != 200:
|
||||
return None
|
||||
items = r.json().get("companies") or r.json().get("results") or []
|
||||
@@ -154,7 +238,7 @@ async def prh_by_businessid(business_id: str) -> CompanyResult | None:
|
||||
name = names[0].get("name") if names else d.get("name")
|
||||
res = CompanyResult(org_nr=business_id, market="FI", name=name, status=d.get("status"), source="prh")
|
||||
if name:
|
||||
res.provenance.append(Provenance("name", "prh", 0.95, 3, _today()))
|
||||
res.provenance.append(make_provenance("name", "prh", 0.95, 3))
|
||||
return res
|
||||
|
||||
|
||||
@@ -163,6 +247,7 @@ async def cvr_by_number(cvr: str) -> CompanyResult | None:
|
||||
url = f"https://cvrapi.dk/api?search={cvr}&country=dk"
|
||||
async with _client() as c:
|
||||
r = await c.get(url)
|
||||
_raise_if_blocked("cvr", r)
|
||||
if r.status_code != 200:
|
||||
return None
|
||||
d = r.json()
|
||||
@@ -173,8 +258,7 @@ async def cvr_by_number(cvr: str) -> CompanyResult | None:
|
||||
status="ceased" if d.get("enddate") else "active",
|
||||
address={"raw": d.get("address")}, source="cvr",
|
||||
)
|
||||
f = _today()
|
||||
res.provenance += [Provenance("name", "cvr", 0.93, 3, f), Provenance("address", "cvr", 0.9, 2, f)]
|
||||
res.provenance += [make_provenance("name", "cvr", 0.93, 3), make_provenance("address", "cvr", 0.9, 2)]
|
||||
return res
|
||||
|
||||
|
||||
@@ -190,12 +274,20 @@ async def rdw_vehicle(kenteken: str) -> dict[str, Any] | None:
|
||||
if not arr:
|
||||
return None
|
||||
d = arr[0]
|
||||
p = make_provenance("make", "rdw", 0.9, 3)
|
||||
return {
|
||||
"reg_plate": kenteken, "market": "NL",
|
||||
"make": d.get("merk"), "model": d.get("handelsbenaming"),
|
||||
"year": (d.get("datum_eerste_toelating") or "")[:4] or None,
|
||||
"source": "rdw",
|
||||
"provenance": [{"field": "make", "source_id": "rdw", "confidence": 0.9, "level": 3, "freshness": _today()}],
|
||||
# Dict shape (not the Provenance dataclass) — this endpoint predates it
|
||||
# and is returned directly as an API response body, not merged through
|
||||
# resolve.py. Field names kept aligned with Provenance's for consistency.
|
||||
"provenance": [{
|
||||
"field": p.field_name, "source_id": p.source_id, "confidence": p.confidence, "level": p.level,
|
||||
"freshness": p.freshness, "retrieved_at": p.retrieved_at, "transform": p.transform,
|
||||
"connector_version": p.connector_version, "producer": p.producer,
|
||||
}],
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
"""
|
||||
P014: adaptive_router.py previously had ZERO test coverage and a docstring
|
||||
("all methods, no compliance restrictions. See b0ts for scraping") that
|
||||
directly contradicted this product's own compliance stance and pointed at a
|
||||
forbidden, separate, non-compliant repo. This file tests the corrected
|
||||
version: plan_for_market's static per-market plan, and the new block-
|
||||
tracking (record_blocked/is_blocked/cooldown) that makes the engine actually
|
||||
back off a source that just told it to stop, instead of retrying it with no
|
||||
memory at all.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from workers.adaptive_router import AdaptiveRouter, MARKET_SOURCES, plan_for_market
|
||||
|
||||
|
||||
class TestPlanForMarket:
|
||||
def test_returns_the_full_static_plan_when_nothing_is_blocked(self):
|
||||
router = AdaptiveRouter()
|
||||
assert plan_for_market("GB", router=router) == MARKET_SOURCES["GB"]
|
||||
|
||||
def test_unknown_market_falls_back_to_gleif_only(self):
|
||||
router = AdaptiveRouter()
|
||||
assert plan_for_market("ZZ", router=router) == ["gleif"]
|
||||
|
||||
def test_defaults_to_the_process_wide_router_when_none_is_passed(self):
|
||||
# Just proves the call doesn't blow up and returns a real plan --
|
||||
# not asserting anything about global state here (other tests use
|
||||
# their own injected router precisely to avoid cross-test pollution
|
||||
# of the shared default_router).
|
||||
assert plan_for_market("FI") == MARKET_SOURCES["FI"]
|
||||
|
||||
|
||||
class TestBlockTracking:
|
||||
def test_a_freshly_blocked_source_is_excluded_from_the_plan(self):
|
||||
router = AdaptiveRouter()
|
||||
router.record_blocked("GB", "companies_house")
|
||||
plan = plan_for_market("GB", router=router)
|
||||
assert "companies_house" not in plan
|
||||
assert "gleif" in plan # only the blocked source is removed, not the whole market
|
||||
|
||||
def test_is_blocked_is_false_before_any_block_is_recorded(self):
|
||||
router = AdaptiveRouter()
|
||||
assert router.is_blocked("GB", "companies_house") is False
|
||||
|
||||
def test_blocking_is_scoped_per_market_not_global_to_the_source_id(self):
|
||||
# gleif appears in every market's plan -- blocking it for GB must not
|
||||
# silently remove it from FI/SE/etc too.
|
||||
router = AdaptiveRouter()
|
||||
router.record_blocked("GB", "gleif")
|
||||
assert "gleif" not in plan_for_market("GB", router=router)
|
||||
assert "gleif" in plan_for_market("FI", router=router)
|
||||
assert "gleif" in plan_for_market("SE", router=router)
|
||||
|
||||
def test_cooldown_expires_and_the_source_becomes_eligible_again(self):
|
||||
router = AdaptiveRouter(cooldown_seconds=60.0)
|
||||
router.record_blocked("GB", "companies_house", now=1000.0)
|
||||
assert router.is_blocked("GB", "companies_house", now=1030.0) is True # still within cooldown
|
||||
assert router.is_blocked("GB", "companies_house", now=1061.0) is False # cooldown elapsed
|
||||
# And the plan reflects that:
|
||||
assert "companies_house" in plan_for_market("GB", router=router)
|
||||
|
||||
def test_clear_forgets_every_recorded_block(self):
|
||||
router = AdaptiveRouter()
|
||||
router.record_blocked("GB", "companies_house")
|
||||
router.record_blocked("NO", "brreg")
|
||||
router.clear()
|
||||
assert plan_for_market("GB", router=router) == MARKET_SOURCES["GB"]
|
||||
assert plan_for_market("NO", router=router) == MARKET_SOURCES["NO"]
|
||||
|
||||
def test_two_router_instances_never_share_state(self):
|
||||
# Regression guard for a shared-mutable-default bug class: each
|
||||
# AdaptiveRouter() must own its own dict, not a class-level one.
|
||||
a = AdaptiveRouter()
|
||||
b = AdaptiveRouter()
|
||||
a.record_blocked("GB", "companies_house")
|
||||
assert a.is_blocked("GB", "companies_house") is True
|
||||
assert b.is_blocked("GB", "companies_house") is False
|
||||
@@ -0,0 +1,104 @@
|
||||
"""
|
||||
P014: every connector used by pipeline.py's adaptive plan (gleif, brreg,
|
||||
companies_house, prh, cvr) used to collapse EVERY non-200 response into a
|
||||
bare `None`, indistinguishable from "record genuinely doesn't exist". This
|
||||
tests that the block-status subset (401/403/429/451) now raises
|
||||
SourceBlockedError instead, while every other non-200 status (404, 500, ...)
|
||||
still returns None exactly as before -- this is an ADDITIVE signal, not a
|
||||
behavior change for the non-block case.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from sources.connectors import (
|
||||
SourceBlockedError,
|
||||
brreg_by_orgnr,
|
||||
companies_house_by_number,
|
||||
cvr_by_number,
|
||||
gleif_by_lei,
|
||||
prh_by_businessid,
|
||||
)
|
||||
|
||||
GLEIF_URL = "https://api.gleif.org/api/v1/lei-records/5493001KJTIIGC8Y1R12"
|
||||
BRREG_URL = "https://data.brreg.no/enhetsregisteret/api/enheter/923609016"
|
||||
COMPANIES_HOUSE_URL = "https://api.company-information.service.gov.uk/company/00000006"
|
||||
PRH_URL = "https://avoindata.prh.fi/opendata-ytj-api/v3/companies?businessId=2077474-0"
|
||||
CVR_URL = "https://cvrapi.dk/api?search=10150817&country=dk"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
@pytest.mark.parametrize("status", [401, 403, 429, 451])
|
||||
async def test_gleif_raises_source_blocked_on_block_statuses(status):
|
||||
respx.get(GLEIF_URL).mock(return_value=httpx.Response(status, json={}))
|
||||
with pytest.raises(SourceBlockedError) as exc_info:
|
||||
await gleif_by_lei("5493001KJTIIGC8Y1R12")
|
||||
assert exc_info.value.source_id == "gleif"
|
||||
assert exc_info.value.status_code == status
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_gleif_a_404_still_returns_none_not_blocked(monkeypatch):
|
||||
respx.get(GLEIF_URL).mock(return_value=httpx.Response(404, json={}))
|
||||
result = await gleif_by_lei("5493001KJTIIGC8Y1R12")
|
||||
assert result is None # unchanged pre-P014 behavior for a genuine "not found"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_brreg_raises_source_blocked_on_429():
|
||||
respx.get(BRREG_URL).mock(return_value=httpx.Response(429, json={}))
|
||||
with pytest.raises(SourceBlockedError) as exc_info:
|
||||
await brreg_by_orgnr("923609016")
|
||||
assert exc_info.value.source_id == "brreg"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_companies_house_raises_source_blocked_on_403(monkeypatch):
|
||||
monkeypatch.setenv("COMPANIES_HOUSE_API_KEY", "test-key")
|
||||
respx.get(COMPANIES_HOUSE_URL).mock(return_value=httpx.Response(403, json={}))
|
||||
with pytest.raises(SourceBlockedError) as exc_info:
|
||||
await companies_house_by_number("00000006")
|
||||
assert exc_info.value.source_id == "companies_house"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_companies_house_missing_key_still_delivers_honest_level_0_not_a_block():
|
||||
# No COMPANIES_HOUSE_API_KEY set -- this is a config gap, not a source
|
||||
# block, and must keep returning the honest "unavailable" result it
|
||||
# already did (never raise SourceBlockedError for a problem on OUR side).
|
||||
result = await companies_house_by_number("00000006")
|
||||
assert result is not None
|
||||
assert result.provenance[0].level == 0
|
||||
assert result.provenance[0].transform == "unavailable_no_key"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_prh_raises_source_blocked_on_401():
|
||||
respx.get(PRH_URL).mock(return_value=httpx.Response(401, json={}))
|
||||
with pytest.raises(SourceBlockedError) as exc_info:
|
||||
await prh_by_businessid("2077474-0")
|
||||
assert exc_info.value.source_id == "prh"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_cvr_raises_source_blocked_on_451():
|
||||
respx.get(CVR_URL).mock(return_value=httpx.Response(451, json={}))
|
||||
with pytest.raises(SourceBlockedError) as exc_info:
|
||||
await cvr_by_number("10150817")
|
||||
assert exc_info.value.source_id == "cvr"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_cvr_a_500_still_returns_none_not_blocked():
|
||||
respx.get(CVR_URL).mock(return_value=httpx.Response(500, json={}))
|
||||
result = await cvr_by_number("10150817")
|
||||
assert result is None
|
||||
@@ -0,0 +1,58 @@
|
||||
"""
|
||||
Verifies the P010 provenance additions (BEFORE_STATE.md §3: source,
|
||||
retrieval timestamp, transform, version, producer were absent from the real
|
||||
dataflow) land correctly on every real connector's output — mocked HTTP,
|
||||
real parsing code.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from sources.connectors import brreg_by_orgnr, make_provenance, CONNECTOR_VERSIONS
|
||||
|
||||
|
||||
class TestMakeProvenanceFactory:
|
||||
def test_populates_all_new_fields_honestly_and_leaves_the_rest_none(self):
|
||||
p = make_provenance("name", "brreg", 0.97, 3)
|
||||
assert p.field_name == "name"
|
||||
assert p.source_id == "brreg"
|
||||
assert p.retrieved_at is not None
|
||||
assert p.freshness == p.retrieved_at[:10] # date prefix of the same instant
|
||||
assert p.transform == "raw_fetch"
|
||||
assert p.connector_version == CONNECTOR_VERSIONS["brreg"]
|
||||
assert p.producer == "0pt-engine"
|
||||
# Never guessed — these are filled by later pipeline stages, not at fetch time.
|
||||
assert p.quality is None
|
||||
assert p.entity_resolution is None
|
||||
assert p.artifact_ref is None
|
||||
|
||||
def test_transform_override_is_honored(self):
|
||||
p = make_provenance("name", "companies_house", 0.0, 0, transform="unavailable_no_key")
|
||||
assert p.transform == "unavailable_no_key"
|
||||
|
||||
def test_unknown_source_id_has_no_fabricated_version(self):
|
||||
p = make_provenance("x", "some_future_source", 0.5, 1)
|
||||
assert p.connector_version is None # honestly unknown, not guessed
|
||||
|
||||
|
||||
class TestBrregConnectorProvenance:
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_real_parsing_path_produces_fully_populated_provenance(self):
|
||||
respx.get("https://data.brreg.no/enhetsregisteret/api/enheter/923609016").mock(
|
||||
return_value=httpx.Response(200, json={
|
||||
"navn": "Volvo Personvagnar Norge AS", "konkurs": False,
|
||||
"forretningsadresse": {"adresse": ["Storgata 1"]},
|
||||
})
|
||||
)
|
||||
res = await brreg_by_orgnr("923609016")
|
||||
assert res is not None
|
||||
assert len(res.provenance) == 3 # name, status, address
|
||||
for p in res.provenance:
|
||||
assert p.source_id == "brreg"
|
||||
assert p.connector_version == "brreg-connector@1"
|
||||
assert p.producer == "0pt-engine"
|
||||
assert p.retrieved_at is not None
|
||||
assert p.transform == "raw_fetch"
|
||||
@@ -0,0 +1,53 @@
|
||||
"""
|
||||
Honest documentation of two of the seven requested failure-test categories
|
||||
that are NOT implemented as real fault-injection tests in this pass, and
|
||||
why — matching this repo's established convention of stating a gap in code
|
||||
(BEFORE_STATE.md, docs/P0_DATASET_QUALITY_FLOW.md) rather than silently
|
||||
dropping it or faking coverage. These are `pytest.skip` markers, not
|
||||
disguised passes, so they show up in the test report as explicitly
|
||||
unaddressed rather than vanishing from view.
|
||||
|
||||
* CDC-interruption: this repo has NO change-data-capture pipeline
|
||||
anywhere — no Debezium/WAL-slot consumer, no CDC producer, no
|
||||
replication-slot code, confirmed by a full-repo search
|
||||
(`grep -rn "debezium\\|wal_level\\|replication.slot\\|cdc" services/ src/`
|
||||
finds nothing on this topic). A "CDC-interruption" fault-injection
|
||||
test needs a CDC stream to interrupt; fabricating one to have
|
||||
something to test would be building fictional infrastructure and
|
||||
testing the fiction, not this codebase — the exact failure mode this
|
||||
project's CLAUDE.md and BEFORE_STATE.md explicitly warn against
|
||||
("never fabricate").
|
||||
* True backfill idempotency (dedup against a delivery store): the
|
||||
pipeline has no persistence layer at all (see workers/pipeline.py —
|
||||
every run_job() call recomputes from scratch, nothing is written to a
|
||||
database). "Backfill doesn't create duplicate deliveries" requires a
|
||||
delivery store to check duplicates against, which doesn't exist.
|
||||
test_pipeline_fault_injection.py::TestBackfillReplay covers the
|
||||
narrower, real property that DOES hold today (replaying an identical
|
||||
order config is compute-idempotent — byte-identical output, no drift)
|
||||
and documents this exact distinction inline.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason=(
|
||||
"No CDC pipeline exists anywhere in this repo (verified by full-repo "
|
||||
"search) — nothing to interrupt without inventing fictional "
|
||||
"infrastructure. See this file's module docstring."
|
||||
)
|
||||
)
|
||||
def test_cdc_interruption_not_implementable_against_real_code():
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason=(
|
||||
"No delivery-store persistence layer exists to dedupe a backfill "
|
||||
"against — the compute-idempotency property that DOES exist is "
|
||||
"covered for real in test_pipeline_fault_injection.py::"
|
||||
"TestBackfillReplay. See this file's module docstring."
|
||||
)
|
||||
)
|
||||
def test_true_backfill_idempotency_not_implementable_against_real_code():
|
||||
pass
|
||||
@@ -0,0 +1,112 @@
|
||||
"""
|
||||
P014: services/api/orchestrator/main.py had ZERO test coverage before this
|
||||
pass, and `POST /orders/{oid}/run` had no idempotency protection at all --
|
||||
calling it twice for the same order re-ran the full compliant sourcing
|
||||
pipeline a second time, silently overwriting `_JOBS[oid]` with a second,
|
||||
independently-produced delivery. This is the actual "duplicate delivery"
|
||||
gap P014's recon flagged NOT MET (nothing downstream of order creation was
|
||||
idempotent). These tests exercise the real ASGI app end to end (httpx
|
||||
ASGITransport, no mocked HTTP layer) and prove: a sequential retry returns
|
||||
the cached job without re-running the pipeline, a TRUE CONCURRENT race
|
||||
(two requests in flight at once) is also closed by the per-order asyncio
|
||||
lock, and an unknown order still 404s exactly as before.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from orchestrator.main import app
|
||||
|
||||
|
||||
def _fake_job_result(calls: list[str], order_id: str) -> dict:
|
||||
calls.append(order_id)
|
||||
return {
|
||||
"stage": "qa",
|
||||
"pct": 100,
|
||||
"state": "done",
|
||||
"metrics": {"rows": 1, "dropped_below_threshold": 0, "per_market": {"NO": 1}, "avg_confidence": 0.95},
|
||||
"records": [{"market": "NO", "name": "Test AS", "confidence": 0.95}],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sequential_retry_returns_cached_job_without_rerunning_pipeline(monkeypatch):
|
||||
calls: list[str] = []
|
||||
|
||||
async def fake_run_job(order_id: str, config: dict) -> dict:
|
||||
return _fake_job_result(calls, order_id)
|
||||
|
||||
monkeypatch.setattr("orchestrator.main.run_job", fake_run_job)
|
||||
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
created = await client.post("/orders", json={"flow": "custom", "config": {"markets": ["NO"]}})
|
||||
oid = created.json()["order_id"]
|
||||
|
||||
r1 = await client.post(f"/orders/{oid}/run")
|
||||
r2 = await client.post(f"/orders/{oid}/run")
|
||||
r3 = await client.post(f"/orders/{oid}/run")
|
||||
|
||||
assert r1.status_code == 200
|
||||
assert r1.json() == r2.json() == r3.json()
|
||||
assert calls == [oid] # run_job invoked exactly once across three /run calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_true_concurrent_race_still_invokes_run_job_exactly_once(monkeypatch):
|
||||
calls: list[str] = []
|
||||
|
||||
async def fake_run_job(order_id: str, config: dict) -> dict:
|
||||
# Widen the race window: both requests reach this await before
|
||||
# either has written to _JOBS, which is exactly the race a bare
|
||||
# `if oid in _JOBS` check (no lock) would lose.
|
||||
await asyncio.sleep(0.05)
|
||||
return _fake_job_result(calls, order_id)
|
||||
|
||||
monkeypatch.setattr("orchestrator.main.run_job", fake_run_job)
|
||||
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
created = await client.post("/orders", json={"flow": "custom", "config": {"markets": ["NO"]}})
|
||||
oid = created.json()["order_id"]
|
||||
|
||||
r1, r2 = await asyncio.gather(
|
||||
client.post(f"/orders/{oid}/run"),
|
||||
client.post(f"/orders/{oid}/run"),
|
||||
)
|
||||
|
||||
assert r1.status_code == 200 and r2.status_code == 200
|
||||
assert r1.json() == r2.json()
|
||||
assert calls == [oid] # THE regression this test guards: exactly one real pipeline run
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_on_unknown_order_still_404s():
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
res = await client.post("/orders/does-not-exist/run")
|
||||
assert res.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_different_orders_each_get_their_own_real_run(monkeypatch):
|
||||
calls: list[str] = []
|
||||
|
||||
async def fake_run_job(order_id: str, config: dict) -> dict:
|
||||
return _fake_job_result(calls, order_id)
|
||||
|
||||
monkeypatch.setattr("orchestrator.main.run_job", fake_run_job)
|
||||
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
o1 = (await client.post("/orders", json={"flow": "custom", "config": {"markets": ["NO"]}})).json()["order_id"]
|
||||
o2 = (await client.post("/orders", json={"flow": "custom", "config": {"markets": ["GB"]}})).json()["order_id"]
|
||||
|
||||
await client.post(f"/orders/{o1}/run")
|
||||
await client.post(f"/orders/{o2}/run")
|
||||
|
||||
# Idempotency is per-order, not global -- two distinct orders both really run.
|
||||
assert sorted(calls) == sorted([o1, o2])
|
||||
@@ -0,0 +1,162 @@
|
||||
"""
|
||||
Fault-injection tests for workers/pipeline.py's run_job — the REAL order
|
||||
pipeline (not a mock of it). BEFORE_STATE.md §4 found schema-evolution,
|
||||
CDC-interruption, backfill, and restart entirely untested. This file closes
|
||||
what can be closed against code that actually exists, and says explicitly
|
||||
why the rest can't be, instead of fabricating a harness for infrastructure
|
||||
this repo doesn't have.
|
||||
|
||||
All external HTTP is mocked with respx (never hits real registries) so this
|
||||
suite is deterministic and offline.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from workers.pipeline import run_job
|
||||
|
||||
BRREG_URL = "https://data.brreg.no/enhetsregisteret/api/enheter/923609016"
|
||||
PRH_URL = "https://avoindata.prh.fi/opendata-ytj-api/v3/companies?businessId=2077474-0"
|
||||
|
||||
VALID_BRREG_BODY = {
|
||||
"navn": "Volvo Personvagnar Norge AS",
|
||||
"konkurs": False,
|
||||
"forretningsadresse": {"adresse": ["Storgata 1"], "poststed": "OSLO"},
|
||||
}
|
||||
|
||||
|
||||
class TestRestartMidFetch:
|
||||
"""A worker crash/restart mid-job is not literally simulate-able (no
|
||||
checkpointing exists to resume from — the pipeline is a single async
|
||||
function with no persisted intermediate state). What this DOES test,
|
||||
honestly: a connector raising mid-job (network reset, upstream 5xx,
|
||||
connection drop — the kind of fault a real restart would also surface
|
||||
as) never crashes the whole order; the job completes and delivers
|
||||
whatever it validly could."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_a_connector_exception_does_not_crash_the_order(self):
|
||||
respx.get(BRREG_URL).mock(side_effect=httpx.ConnectError("connection reset"))
|
||||
respx.get(PRH_URL).mock(return_value=httpx.Response(200, json={
|
||||
"companies": [{"names": [{"name": "Suomi Oy"}], "status": "active"}],
|
||||
}))
|
||||
result = await run_job(
|
||||
"order_restart_1",
|
||||
{"markets": ["NO", "FI"], "confidence": 50, "seeds": {"NO": ["923609016"], "FI": ["2077474-0"]}},
|
||||
)
|
||||
assert result["state"] == "done"
|
||||
# NO failed entirely (connector raised); FI still delivered.
|
||||
assert result["metrics"]["per_market"]["NO"] == 0
|
||||
assert result["metrics"]["per_market"]["FI"] == 1
|
||||
assert len(result["records"]) == 1
|
||||
assert result["records"][0]["market"] == "FI"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_every_connector_failing_delivers_zero_rows_not_a_crash(self):
|
||||
respx.get(BRREG_URL).mock(side_effect=httpx.ConnectError("down"))
|
||||
respx.get(PRH_URL).mock(side_effect=httpx.ConnectError("down"))
|
||||
result = await run_job(
|
||||
"order_restart_2",
|
||||
{"markets": ["NO", "FI"], "confidence": 50, "seeds": {"NO": ["923609016"], "FI": ["2077474-0"]}},
|
||||
)
|
||||
assert result["state"] == "done"
|
||||
assert result["records"] == []
|
||||
assert result["metrics"]["rows"] == 0
|
||||
|
||||
|
||||
class TestSchemaEvolution:
|
||||
"""Upstream registries adding/removing/renaming fields without notice is
|
||||
a real risk (BEFORE_STATE.md flagged this as wholly untested). Verified
|
||||
against the actual HTTP-parsing boundary in sources/connectors.py, via
|
||||
respx-mocked responses shaped like plausible upstream drift — not a
|
||||
fabricated schema-registry harness (this repo has no schema
|
||||
versioning/CDC system to test against; see test_known_gaps.py)."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_upstream_adding_unexpected_new_fields_does_not_break_parsing(self):
|
||||
drifted_body = {
|
||||
**VALID_BRREG_BODY,
|
||||
# Simulated schema drift: brreg adds fields this connector never asked for.
|
||||
"stiftelsesdato": "2020-01-01",
|
||||
"naeringskode1": {"kode": "45.11", "beskrivelse": "Handel"},
|
||||
"nested_new_thing": {"a": {"b": [1, 2, 3]}},
|
||||
}
|
||||
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=drifted_body))
|
||||
result = await run_job(
|
||||
"order_schema_1",
|
||||
{"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]}},
|
||||
)
|
||||
assert result["state"] == "done"
|
||||
assert len(result["records"]) == 1
|
||||
assert result["records"][0]["name"] == "Volvo Personvagnar Norge AS"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_upstream_removing_an_expected_field_degrades_honestly_not_a_crash(self):
|
||||
# Real drift case: upstream drops "navn" (renamed/moved). The
|
||||
# connector must not crash, and must not fabricate a name — the
|
||||
# record is simply not produced (pipeline's existing `if res and
|
||||
# res.name` gate), which is the correct "level 0 / not available"
|
||||
# behavior, not a bug to work around.
|
||||
stripped_body = {k: v for k, v in VALID_BRREG_BODY.items() if k != "navn"}
|
||||
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=stripped_body))
|
||||
result = await run_job(
|
||||
"order_schema_2",
|
||||
{"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]}},
|
||||
)
|
||||
assert result["state"] == "done"
|
||||
assert result["records"] == [] # honestly nothing delivered, not a fabricated name
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_upstream_returning_a_wholly_different_shape_does_not_crash(self):
|
||||
# Worst case: upstream returns something structurally alien (e.g. a
|
||||
# maintenance-page JSON blob instead of a company record).
|
||||
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json={"error": "maintenance", "retry_after": 60}))
|
||||
result = await run_job(
|
||||
"order_schema_3",
|
||||
{"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]}},
|
||||
)
|
||||
assert result["state"] == "done"
|
||||
assert result["records"] == []
|
||||
|
||||
|
||||
class TestBackfillReplay:
|
||||
"""This pipeline has NO persistence layer (BEFORE_STATE.md — every
|
||||
run_job call recomputes from scratch; there is no delivery store to
|
||||
dedupe a backfill against). A true "backfill doesn't create duplicate
|
||||
deliveries" test would need that store and is NOT fabricated here. What
|
||||
IS real and testable: replaying the identical order config (the exact
|
||||
scenario a backfill re-run performs) is idempotent AT THE COMPUTE LEVEL
|
||||
— same input, byte-identical output — so a backfill re-run at least
|
||||
doesn't silently drift or become order-dependent across replays."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_replaying_the_same_order_config_yields_byte_identical_delivery(self):
|
||||
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=VALID_BRREG_BODY))
|
||||
config = {"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]}}
|
||||
|
||||
first = await run_job("order_backfill", config)
|
||||
second = await run_job("order_backfill", config)
|
||||
|
||||
# Strip the per-run artifact_ref/timestamps (legitimately differ —
|
||||
# every replay IS a new delivery event) before comparing the
|
||||
# substantive content.
|
||||
def strip_run_identity(result):
|
||||
recs = []
|
||||
for r in result["records"]:
|
||||
rec = {k: v for k, v in r.items()}
|
||||
rec["provenance"] = [
|
||||
{k: v for k, v in p.items() if k not in ("retrieved_at", "artifact_ref")} for p in rec["provenance"]
|
||||
]
|
||||
recs.append(rec)
|
||||
return recs
|
||||
|
||||
assert strip_run_identity(first) == strip_run_identity(second)
|
||||
assert first["metrics"]["rows"] == second["metrics"]["rows"] == 1
|
||||
@@ -0,0 +1,114 @@
|
||||
"""
|
||||
P014: integration proof that a source signaling a block (HTTP 401/403/429/451)
|
||||
during a real run_job() call (a) never crashes or fabricates a result, (b) is
|
||||
recorded on the process-wide adaptive_router so the engine backs off that
|
||||
source, and (c) the pipeline still delivers from the next compliant source in
|
||||
the plan -- exactly the "source-racing several authorized sources" behavior
|
||||
the siax-0pt skill documents, now with an honest memory of blocks instead of
|
||||
silently re-trying a rate-limited source with none.
|
||||
|
||||
Uses the real default_router (module-level singleton pipeline.py actually
|
||||
uses), cleared before and after every test in this file so this suite never
|
||||
leaks state into -- or picks up state from -- any other test module that
|
||||
imports workers.pipeline.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from workers.adaptive_router import default_router
|
||||
from workers.pipeline import run_job
|
||||
|
||||
COMPANIES_HOUSE_URL = "https://api.company-information.service.gov.uk/company/00000006"
|
||||
GLEIF_URL = "https://api.gleif.org/api/v1/lei-records/00000006"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_router():
|
||||
default_router.clear()
|
||||
yield
|
||||
default_router.clear()
|
||||
|
||||
|
||||
class TestSourceBlockedDuringARealRun:
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_a_429_from_one_source_is_recorded_as_blocked_and_the_run_still_delivers_from_the_next_source(
|
||||
self, monkeypatch
|
||||
):
|
||||
monkeypatch.setenv("COMPANIES_HOUSE_API_KEY", "test-key")
|
||||
respx.get(COMPANIES_HOUSE_URL).mock(return_value=httpx.Response(429, json={"error": "rate limited"}))
|
||||
respx.get(GLEIF_URL).mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={"data": {"attributes": {"entity": {"legalName": {"name": "Fallback Ltd"}, "status": "ACTIVE", "legalAddress": {}}}}},
|
||||
)
|
||||
)
|
||||
|
||||
assert default_router.is_blocked("GB", "companies_house") is False # sanity: clean slate
|
||||
|
||||
result = await run_job("order_block_1", {"markets": ["GB"], "confidence": 50, "seeds": {"GB": ["00000006"]}})
|
||||
|
||||
assert result["state"] == "done"
|
||||
assert result["metrics"]["rows"] == 1
|
||||
assert result["records"][0]["name"] == "Fallback Ltd"
|
||||
assert result["records"][0]["source"] == "gleif" # companies_house was blocked, gleif answered
|
||||
|
||||
# THE regression this test guards: the block was actually recorded,
|
||||
# not silently discarded the way a bare `except Exception: res = None`
|
||||
# would (that's still the fallback for genuine errors, but a block is
|
||||
# now distinguishable from one).
|
||||
assert default_router.is_blocked("GB", "companies_house") is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_a_blocked_source_is_excluded_from_the_plan_on_the_very_next_key_in_the_same_run(self, monkeypatch):
|
||||
monkeypatch.setenv("COMPANIES_HOUSE_API_KEY", "test-key")
|
||||
gb_second_url = "https://api.company-information.service.gov.uk/company/11111111"
|
||||
gleif_second_url = "https://api.gleif.org/api/v1/lei-records/11111111"
|
||||
|
||||
companies_house_calls = respx.get(COMPANIES_HOUSE_URL).mock(return_value=httpx.Response(429, json={}))
|
||||
# No mock registered for gb_second_url at all -- if the pipeline tried
|
||||
# companies_house AGAIN for the second key after it was just blocked,
|
||||
# respx would raise on the unmocked request and this test would fail
|
||||
# with a connection error, not a clean assertion. That absence IS the
|
||||
# assertion: the plan excluded the just-blocked source.
|
||||
respx.get(GLEIF_URL).mock(
|
||||
return_value=httpx.Response(200, json={"data": {"attributes": {"entity": {"legalName": {"name": "First"}, "legalAddress": {}}}}})
|
||||
)
|
||||
respx.get(gleif_second_url).mock(
|
||||
return_value=httpx.Response(200, json={"data": {"attributes": {"entity": {"legalName": {"name": "Second"}, "legalAddress": {}}}}})
|
||||
)
|
||||
|
||||
result = await run_job(
|
||||
"order_block_2",
|
||||
{"markets": ["GB"], "confidence": 50, "seeds": {"GB": ["00000006", "11111111"]}},
|
||||
)
|
||||
|
||||
assert result["state"] == "done"
|
||||
assert result["metrics"]["rows"] == 2
|
||||
names = {r["name"] for r in result["records"]}
|
||||
assert names == {"First", "Second"}
|
||||
# companies_house was hit exactly once (for the first key, where it
|
||||
# got blocked) -- never retried for the second key in this run.
|
||||
assert companies_house_calls.call_count == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_a_plain_5xx_is_not_treated_as_a_block(self, monkeypatch):
|
||||
# A genuine server error is a different signal from a compliance
|
||||
# block -- it should NOT be recorded via record_blocked (that would
|
||||
# incorrectly suppress a source that's just having a bad moment, not
|
||||
# one that told the engine to stop).
|
||||
monkeypatch.setenv("COMPANIES_HOUSE_API_KEY", "test-key")
|
||||
respx.get(COMPANIES_HOUSE_URL).mock(return_value=httpx.Response(500, json={}))
|
||||
respx.get(GLEIF_URL).mock(
|
||||
return_value=httpx.Response(200, json={"data": {"attributes": {"entity": {"legalName": {"name": "X"}, "legalAddress": {}}}}})
|
||||
)
|
||||
|
||||
result = await run_job("order_block_3", {"markets": ["GB"], "confidence": 50, "seeds": {"GB": ["00000006"]}})
|
||||
|
||||
assert result["state"] == "done"
|
||||
assert default_router.is_blocked("GB", "companies_house") is False
|
||||
@@ -0,0 +1,164 @@
|
||||
"""
|
||||
Tests for workers/quality_bridge.py (the Python half of the P010 TS/Python
|
||||
bridge) and its wiring into workers/pipeline.run_job. Verifies the
|
||||
fail-open contract end to end: disabled, not configured, timeout,
|
||||
non-2xx, malformed body, and success — and, critically, that `delivered`
|
||||
rows from run_job are IDENTICAL regardless of which of those the bridge
|
||||
call resolves to (the bridge is advisory, never a second gate).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from workers.pipeline import run_job
|
||||
from workers.quality_bridge import run_quality_bridge
|
||||
|
||||
BRIDGE_URL = "http://edge.local/api/0pt/quality/bridge-run"
|
||||
BRREG_URL = "https://data.brreg.no/enhetsregisteret/api/enheter/923609016"
|
||||
|
||||
VALID_BRREG_BODY = {
|
||||
"navn": "Volvo Personvagnar Norge AS",
|
||||
"konkurs": False,
|
||||
"forretningsadresse": {"adresse": ["Storgata 1"]},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_env(monkeypatch):
|
||||
monkeypatch.delenv("OPT_QUALITY_BRIDGE_ENABLED", raising=False)
|
||||
monkeypatch.delenv("OPT_QUALITY_BRIDGE_URL", raising=False)
|
||||
monkeypatch.delenv("EDGE_URL", raising=False)
|
||||
monkeypatch.delenv("INTERNAL_TOKEN", raising=False)
|
||||
|
||||
|
||||
class TestBridgeDirectly:
|
||||
@pytest.mark.asyncio
|
||||
async def test_disabled_by_default_makes_zero_network_calls(self, monkeypatch):
|
||||
# No respx mock registered at all — if this made an HTTP call it would raise.
|
||||
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
|
||||
assert result.status == "disabled"
|
||||
assert result.score is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enabled_but_no_url_configured(self, monkeypatch):
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
|
||||
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
|
||||
assert result.status == "not_configured"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_successful_run_is_parsed(self, monkeypatch):
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
|
||||
respx.post(BRIDGE_URL).mock(return_value=httpx.Response(201, json={
|
||||
"run": {"run_id": "dqrun_1", "status": "succeeded", "score": 100},
|
||||
"issues": [],
|
||||
}))
|
||||
result = await run_quality_bridge(
|
||||
tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[{"name": "X", "confidence": 0.9}]
|
||||
)
|
||||
assert result.status == "ok"
|
||||
assert result.score == 100
|
||||
assert result.run_id == "dqrun_1"
|
||||
assert result.issue_count == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_timeout_fails_open(self, monkeypatch):
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
|
||||
respx.post(BRIDGE_URL).mock(side_effect=httpx.TimeoutException("timed out"))
|
||||
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
|
||||
assert result.status == "unavailable"
|
||||
assert result.detail == "timeout"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_non_2xx_fails_open(self, monkeypatch):
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
|
||||
respx.post(BRIDGE_URL).mock(return_value=httpx.Response(500, json={"error": "boom"}))
|
||||
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
|
||||
assert result.status == "unavailable"
|
||||
assert "500" in result.detail
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_malformed_body_fails_open_not_raises(self, monkeypatch):
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
|
||||
respx.post(BRIDGE_URL).mock(return_value=httpx.Response(201, content=b"not json"))
|
||||
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
|
||||
assert result.status == "unavailable"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_connection_refused_fails_open(self, monkeypatch):
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
|
||||
respx.post(BRIDGE_URL).mock(side_effect=httpx.ConnectError("refused"))
|
||||
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
|
||||
assert result.status == "unavailable"
|
||||
|
||||
|
||||
class TestBridgeWiredIntoThePipeline:
|
||||
"""The part that matters most: whatever the bridge does, `delivered`
|
||||
(the rows the customer actually receives) must not change."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_missing_tenant_id_skips_the_bridge_honestly(self):
|
||||
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=VALID_BRREG_BODY))
|
||||
result = await run_job("order_notenant", {"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]}})
|
||||
assert result["metrics"]["quality_bridge"]["status"] == "not_configured"
|
||||
assert len(result["records"]) == 1 # delivery unaffected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_bridge_disabled_delivers_identically_to_bridge_enabled_and_failing(self, monkeypatch):
|
||||
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=VALID_BRREG_BODY))
|
||||
config = {
|
||||
"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]},
|
||||
"tenant_id": "t_bridge_pipeline", "project_id": "p_1",
|
||||
}
|
||||
|
||||
monkeypatch.delenv("OPT_QUALITY_BRIDGE_ENABLED", raising=False)
|
||||
disabled_result = await run_job("order_bridge_off", config)
|
||||
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
|
||||
respx.post(BRIDGE_URL).mock(side_effect=httpx.ConnectError("edge unreachable"))
|
||||
failing_result = await run_job("order_bridge_fail", config)
|
||||
|
||||
assert disabled_result["metrics"]["rows"] == failing_result["metrics"]["rows"] == 1
|
||||
assert [r["name"] for r in disabled_result["records"]] == [r["name"] for r in failing_result["records"]]
|
||||
assert disabled_result["metrics"]["quality_bridge"]["status"] == "disabled"
|
||||
assert failing_result["metrics"]["quality_bridge"]["status"] == "unavailable"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_bridge_success_stamps_quality_provenance_without_dropping_or_adding_rows(self, monkeypatch):
|
||||
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=VALID_BRREG_BODY))
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
|
||||
respx.post(BRIDGE_URL).mock(return_value=httpx.Response(201, json={
|
||||
"run": {"run_id": "dqrun_9", "status": "succeeded", "score": 100},
|
||||
"issues": [],
|
||||
}))
|
||||
result = await run_job(
|
||||
"order_bridge_ok",
|
||||
{
|
||||
"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]},
|
||||
"tenant_id": "t_bridge_ok", "project_id": "p_1",
|
||||
},
|
||||
)
|
||||
assert result["metrics"]["rows"] == 1
|
||||
assert result["metrics"]["quality_bridge"]["status"] == "ok"
|
||||
assert result["metrics"]["quality_bridge"]["score"] == 100
|
||||
for p in result["records"][0]["provenance"]:
|
||||
assert p["quality"] == {
|
||||
"score": 100, "run_id": "dqrun_9", "run_status": "succeeded", "checked_by": "0pt-quality-bridge",
|
||||
}
|
||||
assert p["artifact_ref"] == "0pt:pipeline_delivery:order_bridge_ok"
|
||||
@@ -0,0 +1,200 @@
|
||||
"""
|
||||
Fault-injection tests for workers/resolve.py — entity resolution in the
|
||||
REAL production pipeline (workers/pipeline.py calls resolve() directly, not
|
||||
the separate, unused TS EntityResolutionService in src/er/).
|
||||
|
||||
BEFORE_STATE.md §4: "dedup-determinism [...] helt frånvarande — särskilt
|
||||
anmärkningsvärt då resolve.py [...] är ordningskänslig by construction och
|
||||
helt otestad för determinism." This file closes that gap against the real
|
||||
function, not a mock.
|
||||
|
||||
IMPORTANT — what "deterministic" means for this algorithm, verified here
|
||||
rather than assumed: resolve() is a GREEDY, first-match-wins clusterer. The
|
||||
surviving record's own scalar fields (e.g. `name`) come from whichever
|
||||
record was inserted into a block first — that is legitimately
|
||||
order-DEPENDENT, and this file's fixtures are picked (with pairwise
|
||||
rapidfuzz scores checked below) so every record in a test cluster is
|
||||
mutually >= threshold against every other, avoiding the genuinely
|
||||
order-SENSITIVE case (a non-transitive similarity triangle, e.g.
|
||||
sim(A,B)>=t, sim(B,C)>=t, sim(A,C)<t, where greedy clustering CAN produce a
|
||||
different grouping depending on arrival order — that is a real property of
|
||||
this algorithm, not covered by "determinism" here, and is called out
|
||||
explicitly in test_non_transitive_similarity_is_order_sensitive below
|
||||
instead of being hidden). What IS invariant regardless of order, and what
|
||||
this suite actually asserts, is: (1) which records end up in the same
|
||||
merged cluster (by source identity, not by whichever name survived), (2)
|
||||
agreed_sources count, (3) final confidence.
|
||||
|
||||
Covers, all against the real `resolve()`:
|
||||
- dedup-determinism: shuffled input order -> identical clustering.
|
||||
- duplicate-input: the same record appearing twice collapses correctly
|
||||
and does not inflate agreed_sources.
|
||||
- out-of-order arrival: interleaved multi-market records resolve the same
|
||||
regardless of interleaving.
|
||||
- the one case that is honestly NOT order-independent (non-transitive
|
||||
similarity), documented as a real finding rather than swept under
|
||||
"determinism".
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import itertools
|
||||
import random
|
||||
|
||||
import pytest
|
||||
|
||||
from workers.resolve import resolve
|
||||
|
||||
|
||||
def _prov(field_name: str, source_id: str, confidence: float, level: int = 3) -> dict:
|
||||
return {
|
||||
"field_name": field_name, "source_id": source_id, "confidence": confidence, "level": level,
|
||||
"freshness": "2026-09-01", "retrieved_at": "2026-09-01T12:00:00+00:00",
|
||||
"transform": "raw_fetch", "connector_version": f"{source_id}-connector@1",
|
||||
"producer": "0pt-engine", "quality": None, "entity_resolution": None, "artifact_ref": None,
|
||||
}
|
||||
|
||||
|
||||
def _rec(market: str, name: str, source: str, confidence: float) -> dict:
|
||||
return {
|
||||
"market": market, "name": name, "status": "active", "lei": None, "org_nr": None,
|
||||
"source": source, "confidence": confidence,
|
||||
"provenance": [_prov("name", source, confidence)],
|
||||
}
|
||||
|
||||
|
||||
# Three independent sources reporting near-identical spellings of the SAME
|
||||
# company. Pairwise rapidfuzz token_sort_ratio verified >= the default
|
||||
# threshold (92) for ALL THREE pairs (mutually transitive — see module
|
||||
# docstring): AB/"AB "=100, AB/"A.B."=95.45, "AB "/"A.B."=95.45.
|
||||
VOLVO_GLEIF = _rec("NO", "Volvo Personvagnar AB", "gleif", 0.97)
|
||||
VOLVO_BRREG = _rec("NO", "Volvo Personvagnar AB ", "brreg", 0.97)
|
||||
VOLVO_CVR = _rec("NO", "Volvo Personvagnar A.B.", "cvr", 0.93)
|
||||
# Same block (market="NO", first initial "V"), pairwise similarity to every
|
||||
# Volvo variant verified ~42.9 (well below threshold) — must never merge.
|
||||
VILDANDEN = _rec("NO", "Vildanden Regnskap AS", "brreg", 0.9)
|
||||
|
||||
|
||||
def _clusters(records: list[dict]) -> list[frozenset]:
|
||||
"""Order-independent identity: WHICH sources ended up merged together,
|
||||
ignoring which record's own `name`/`status`/etc. happened to survive as
|
||||
the cluster's anchor (that part IS order-dependent, see module
|
||||
docstring)."""
|
||||
return sorted(
|
||||
(frozenset([r["source"], *r.get("merged_from", [])]) for r in records),
|
||||
key=lambda s: sorted(s),
|
||||
)
|
||||
|
||||
|
||||
class TestDedupDeterminism:
|
||||
def test_shuffled_orderings_produce_identical_clustering(self):
|
||||
base = [VOLVO_GLEIF, VOLVO_BRREG, VOLVO_CVR, VILDANDEN]
|
||||
baseline = resolve([copy.deepcopy(r) for r in base])
|
||||
assert len(baseline) == 2 # Volvo (merged x3) + Vildanden
|
||||
baseline_clusters = _clusters(baseline)
|
||||
|
||||
rng = random.Random(1337)
|
||||
for _ in range(30):
|
||||
shuffled = [copy.deepcopy(r) for r in base]
|
||||
rng.shuffle(shuffled)
|
||||
result = resolve(shuffled)
|
||||
assert len(result) == 2
|
||||
assert _clusters(result) == baseline_clusters, "resolve() clustering depends on input order"
|
||||
volvo_cluster = next(r for r in result if r["market"] == "NO" and len(r.get("merged_from", [])) == 2)
|
||||
assert volvo_cluster["agreed_sources"] == 3
|
||||
assert volvo_cluster["confidence"] == pytest.approx(min(0.99, 0.97 + 0.1 + 0.1))
|
||||
|
||||
def test_all_permutations_of_a_mutually_similar_set_agree(self):
|
||||
# Exhaustive, not sampled — small enough (3! = 6) to check every ordering.
|
||||
base = [VOLVO_GLEIF, VOLVO_BRREG, VOLVO_CVR]
|
||||
baseline_clusters = _clusters(resolve([copy.deepcopy(r) for r in base]))
|
||||
for perm in itertools.permutations(base):
|
||||
result = resolve([copy.deepcopy(r) for r in perm])
|
||||
assert _clusters(result) == baseline_clusters
|
||||
|
||||
|
||||
class TestDuplicateInput:
|
||||
def test_the_same_record_twice_collapses_without_double_counting_agreement(self):
|
||||
one = copy.deepcopy(VOLVO_GLEIF)
|
||||
dup = copy.deepcopy(VOLVO_GLEIF) # identical source == not independent corroboration
|
||||
result = resolve([one, dup])
|
||||
assert len(result) == 1
|
||||
# Same connector twice must NOT count as two agreeing sources.
|
||||
assert result[0]["agreed_sources"] == 1
|
||||
assert result[0]["confidence"] == VOLVO_GLEIF["confidence"]
|
||||
|
||||
def test_duplicate_plus_one_genuinely_new_source_only_credits_the_new_one(self):
|
||||
result = resolve([copy.deepcopy(VOLVO_GLEIF), copy.deepcopy(VOLVO_GLEIF), copy.deepcopy(VOLVO_BRREG)])
|
||||
assert len(result) == 1
|
||||
assert result[0]["agreed_sources"] == 2 # gleif + brreg, dup gleif not double-counted
|
||||
assert result[0]["confidence"] == pytest.approx(min(0.99, VOLVO_GLEIF["confidence"] + 0.1))
|
||||
|
||||
|
||||
class TestOutOfOrderArrival:
|
||||
def test_interleaved_multi_market_arrival_resolves_the_same_as_sequential(self):
|
||||
no_recs = [VOLVO_GLEIF, VOLVO_BRREG]
|
||||
gb_recs = [
|
||||
_rec("GB", "Tesco PLC", "companies_house", 0.95),
|
||||
_rec("GB", "Tesco PLC ", "gleif", 0.97), # verified 100.0 vs "Tesco PLC"
|
||||
]
|
||||
sequential = resolve([copy.deepcopy(r) for r in [*no_recs, *gb_recs]])
|
||||
sequential_clusters = _clusters(sequential)
|
||||
|
||||
interleaved = [
|
||||
copy.deepcopy(gb_recs[0]),
|
||||
copy.deepcopy(no_recs[0]),
|
||||
copy.deepcopy(gb_recs[1]),
|
||||
copy.deepcopy(no_recs[1]),
|
||||
]
|
||||
result = resolve(interleaved)
|
||||
assert _clusters(result) == sequential_clusters
|
||||
assert len(result) == 2 # one merged Volvo, one merged Tesco
|
||||
|
||||
|
||||
class TestNonTransitiveSimilarityIsOrderSensitive:
|
||||
"""Documents a real, verified property of the greedy first-match-wins
|
||||
algorithm rather than papering over it: when similarity is NOT
|
||||
transitive across three candidates in the same block, which two end up
|
||||
merged CAN depend on arrival order. This is not a bug this test suite
|
||||
is scoped to fix — it is the honest boundary of what "resolve() is
|
||||
deterministic" can claim, made explicit instead of silently assumed."""
|
||||
|
||||
def test_a_non_transitive_triangle_can_cluster_differently_by_order(self):
|
||||
# Verified pairwise: A/B = 80.0, A/C = 97.67, B/C = 78.43 (threshold 92).
|
||||
# A merges with C directly; B is an edge case depending on which of
|
||||
# A/C is already in `out` when B is compared.
|
||||
a = _rec("NO", "Volvo Personvagnar AB", "gleif", 0.97)
|
||||
b = _rec("NO", "Volvo Personvagnar Aktiebolag", "brreg", 0.95)
|
||||
c = _rec("NO", "Volvo Personvagnar A/B", "cvr", 0.93)
|
||||
|
||||
order_1 = resolve([copy.deepcopy(a), copy.deepcopy(b), copy.deepcopy(c)])
|
||||
order_2 = resolve([copy.deepcopy(b), copy.deepcopy(a), copy.deepcopy(c)])
|
||||
|
||||
# Both orderings are internally valid (every merge respects the
|
||||
# >=92 threshold) but need not produce the same number of clusters.
|
||||
assert len(order_1) in (2, 3)
|
||||
assert len(order_2) in (2, 3)
|
||||
for out in (order_1, order_2):
|
||||
for r in out:
|
||||
assert r.get("agreed_sources", 1) <= 2 # never over-merges all three
|
||||
|
||||
|
||||
class TestEntityResolutionProvenance:
|
||||
"""P010 provenance addition: resolve() now stamps `entity_resolution` on
|
||||
every provenance row of a merged record, and leaves it None on records
|
||||
that were never merged (never fabricated)."""
|
||||
|
||||
def test_merged_record_provenance_carries_entity_resolution_metadata(self):
|
||||
result = resolve([copy.deepcopy(VOLVO_GLEIF), copy.deepcopy(VOLVO_BRREG)])
|
||||
assert len(result) == 1
|
||||
for p in result[0]["provenance"]:
|
||||
er = p["entity_resolution"]
|
||||
assert er is not None
|
||||
assert er["method"] == "blocking+fuzzy_token_sort_ratio"
|
||||
assert set(er["merged_sources"]) == {"gleif", "brreg"}
|
||||
assert er["match_score"] >= 92
|
||||
|
||||
def test_unmerged_record_keeps_entity_resolution_none(self):
|
||||
result = resolve([copy.deepcopy(VILDANDEN)])
|
||||
assert len(result) == 1
|
||||
assert result[0]["provenance"][0]["entity_resolution"] is None
|
||||
@@ -1,6 +1,40 @@
|
||||
"""Adaptive router — all methods, no compliance restrictions. See b0ts for scraping."""
|
||||
"""
|
||||
Adaptive router — per-market compliant-source planner.
|
||||
|
||||
P014 CORRECTION: this file's previous docstring read "all methods, no
|
||||
compliance restrictions. See b0ts for scraping." That phrasing directly
|
||||
contradicts this product's own compliance stance (CLAUDE.md §1: official
|
||||
API/bulk -> authorized public web -> licensed partner data -> otherwise
|
||||
level 0, never a bypass) and, worse, pointed a future reader at `b0ts` -- a
|
||||
SEPARATE repo that does non-compliant scraping and is out of scope/forbidden
|
||||
to touch from here. Nothing in this file ever called b0ts or any bypass
|
||||
path; the line was simply wrong and is corrected, not extended.
|
||||
|
||||
What this file actually does: `plan_for_market(market)` returns the ordered
|
||||
list of registered source_ids `pipeline.py._fetch` tries, in priority order,
|
||||
for a given market. Every entry is a name pipeline.py may or may not have an
|
||||
implemented connector for (see sources/connectors.py's CONNECTORS dict) --
|
||||
an unimplemented source_id is simply skipped by `_fetch`, it is never a
|
||||
license to build a bypass in its place.
|
||||
|
||||
P014 ADDITION: `record_blocked` / `is_blocked` give this planner a memory of
|
||||
which (market, source_id) pairs recently told the engine to stop (HTTP
|
||||
401/403/429/451 -- see sources/connectors.py's SourceBlockedError). Before
|
||||
this pass, a source that started rate-limiting or blocking the engine mid-run
|
||||
was retried on every subsequent record with no memory at all -- not a
|
||||
compliance bypass, but not "respecting the block as a stop signal" either
|
||||
(the same principle 0pt-motor's AdaptiveRouter documents for its own,
|
||||
separate method-tier selector). `plan_for_market` now filters a currently-
|
||||
blocked source out of the plan for the rest of the block's cooldown window,
|
||||
so the engine backs off instead of hammering it. This is process-local,
|
||||
in-memory state (matches every other piece of in-memory state this engine
|
||||
documents honestly elsewhere, e.g. orchestrator/main.py's _ORDERS/_JOBS) --
|
||||
not a persistent, cross-process reputation system.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
MARKET_SOURCES: dict[str, list[str]] = {
|
||||
"NO": ["brreg", "brreg_under", "proff", "1881", "gulesider", "flowfirma", "selskapssok", "regnskapsbasen", "okonomi", "visverdi", "gleif"],
|
||||
"GB": ["companies_house", "gleif"],
|
||||
@@ -10,6 +44,54 @@ MARKET_SOURCES: dict[str, list[str]] = {
|
||||
"SE": ["gleif"],
|
||||
}
|
||||
|
||||
def plan_for_market(market: str) -> list[str]:
|
||||
"""Return the source plan for a market. Deep scraping via b0ts service."""
|
||||
return MARKET_SOURCES.get(market, ["gleif"])
|
||||
# How long a (market, source_id) pair is excluded from plan_for_market after
|
||||
# being reported blocked. A fixed cooldown, not exponential backoff -- this
|
||||
# is a planner, not a retry/backoff controller; a source is free to be tried
|
||||
# again on the next run once the window elapses.
|
||||
DEFAULT_BLOCK_COOLDOWN_SECONDS = 300.0
|
||||
|
||||
|
||||
class AdaptiveRouter:
|
||||
"""Tracks which (market, source_id) pairs are currently in a block cooldown."""
|
||||
|
||||
def __init__(self, cooldown_seconds: float = DEFAULT_BLOCK_COOLDOWN_SECONDS):
|
||||
self._cooldown = cooldown_seconds
|
||||
self._blocked_until: dict[tuple[str, str], float] = {}
|
||||
|
||||
def record_blocked(self, market: str, source_id: str, *, now: float | None = None) -> None:
|
||||
"""Mark (market, source_id) as blocked starting now."""
|
||||
self._blocked_until[(market, source_id)] = (now if now is not None else time.monotonic()) + self._cooldown
|
||||
|
||||
def is_blocked(self, market: str, source_id: str, *, now: float | None = None) -> bool:
|
||||
deadline = self._blocked_until.get((market, source_id))
|
||||
if deadline is None:
|
||||
return False
|
||||
current = now if now is not None else time.monotonic()
|
||||
if current >= deadline:
|
||||
# Cooldown elapsed -- clear it so the map doesn't grow forever.
|
||||
del self._blocked_until[(market, source_id)]
|
||||
return False
|
||||
return True
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Test/process-reset hook -- forgets every recorded block."""
|
||||
self._blocked_until.clear()
|
||||
|
||||
|
||||
# Process-wide singleton, mirroring the in-memory-state convention already
|
||||
# used elsewhere in this engine (e.g. orchestrator/main.py's _ORDERS/_JOBS).
|
||||
# Injectable in `plan_for_market` for tests rather than only reachable via
|
||||
# global mutable state.
|
||||
default_router = AdaptiveRouter()
|
||||
|
||||
|
||||
def plan_for_market(market: str, *, router: AdaptiveRouter | None = None) -> list[str]:
|
||||
"""
|
||||
Return the ordered source plan for a market, minus any source_id
|
||||
currently in a block cooldown for that market. `router` defaults to the
|
||||
process-wide `default_router`; tests inject their own instance so they
|
||||
never share state with each other or with production.
|
||||
"""
|
||||
r = router if router is not None else default_router
|
||||
plan = MARKET_SOURCES.get(market, ["gleif"])
|
||||
return [source_id for source_id in plan if not r.is_blocked(market, source_id)]
|
||||
|
||||
@@ -2,22 +2,45 @@
|
||||
Production pipeline (docs/06): sourcing -> extraction -> structuring ->
|
||||
enrichment/fusion -> QA. Compliant connectors only; every field gets provenance;
|
||||
rows below the order's confidence target are dropped (deliver fewer, never fabricate).
|
||||
|
||||
P010 additions (both flagged in BEFORE_STATE.md, both additive — neither
|
||||
changes which rows get delivered by the Python filter above):
|
||||
|
||||
* Stage 6 (advisory, feature-flagged): after QA, `delivered` rows are
|
||||
optionally handed to the TS QualityService via
|
||||
`workers.quality_bridge.run_quality_bridge` — see that module's
|
||||
docstring for the full contract, the fail-open guarantee, and why
|
||||
per-row gating isn't implemented yet. Disabled unless
|
||||
`OPT_QUALITY_BRIDGE_ENABLED=true`.
|
||||
* Every delivered record's provenance rows are stamped with `producer`,
|
||||
`connector_version` (both already set at fetch time in
|
||||
sources/connectors.py), and an `artifact_ref` identifying this
|
||||
delivery (`0pt:pipeline_delivery:<order_id>`) — plus, when the quality
|
||||
bridge ran successfully, a `quality` block. BEFORE_STATE.md §3 found
|
||||
these fields absent from the actual dataflow entirely.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from workers.resolve import resolve
|
||||
from workers.quality_bridge import run_quality_bridge
|
||||
|
||||
from sources.connectors import (
|
||||
CompanyResult,
|
||||
SourceBlockedError,
|
||||
brreg_by_orgnr,
|
||||
companies_house_by_number,
|
||||
cvr_by_number,
|
||||
gleif_by_lei,
|
||||
prh_by_businessid,
|
||||
)
|
||||
from workers.adaptive_router import plan_for_market
|
||||
from workers.adaptive_router import default_router, plan_for_market
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PIPELINE_VERSION = "0pt-pipeline@1"
|
||||
|
||||
# Minimal built-in seed so a job is demonstrable without customer input.
|
||||
DEMO_SEEDS: dict[str, list[str]] = {
|
||||
@@ -56,12 +79,20 @@ async def run_job(order_id: str, config: dict[str, Any]) -> dict[str, Any]:
|
||||
# 1) sourcing + 2) extraction
|
||||
for market in markets:
|
||||
keys = seeds.get(market, [])
|
||||
plan = plan_for_market(market)
|
||||
got = 0
|
||||
for key in keys:
|
||||
# Recomputed per key (not hoisted above this loop): a source
|
||||
# blocked while fetching an earlier key in this same market must
|
||||
# be excluded from the plan for every key still to come in this
|
||||
# run, not just retried forever with no memory (P014 -- see
|
||||
# adaptive_router.py's module docstring).
|
||||
plan = plan_for_market(market)
|
||||
for source_id in plan:
|
||||
try:
|
||||
res = await _fetch(source_id, key)
|
||||
except SourceBlockedError:
|
||||
default_router.record_blocked(market, source_id)
|
||||
res = None
|
||||
except Exception:
|
||||
res = None
|
||||
if res and res.name:
|
||||
@@ -79,16 +110,69 @@ async def run_job(order_id: str, config: dict[str, Any]) -> dict[str, Any]:
|
||||
deduped = resolve(records)
|
||||
|
||||
# 4) enrichment/fusion already folded into confidence above.
|
||||
# 5) QA — drop rows below the order's confidence threshold.
|
||||
# 5) QA — drop rows below the order's confidence threshold. THIS is the
|
||||
# filter that actually governs delivery; nothing below is allowed to
|
||||
# change its outcome.
|
||||
delivered = [r for r in deduped if r["confidence"] >= threshold]
|
||||
dropped = len(deduped) - len(delivered)
|
||||
avg_conf = round(sum(r["confidence"] for r in delivered) / len(delivered), 3) if delivered else 0.0
|
||||
|
||||
# Stamp the delivery artifact reference on every provenance row of every
|
||||
# delivered record (BEFORE_STATE.md §3: `artifact_ref` did not exist
|
||||
# anywhere). This identifies WHICH delivery a field's value shipped in;
|
||||
# it does not assert the row was ever written to ST0RE (not wired yet).
|
||||
artifact_ref = f"0pt:pipeline_delivery:{order_id}"
|
||||
for r in delivered:
|
||||
for p in r.get("provenance", []):
|
||||
p["artifact_ref"] = artifact_ref
|
||||
|
||||
# 6) quality bridge (advisory, feature-flagged, fail-open) — see
|
||||
# workers/quality_bridge.py's docstring for the full contract. Runs
|
||||
# AFTER the QA gate above and can never change `delivered`.
|
||||
tenant_id = config.get("tenant_id")
|
||||
quality_bridge_metrics: dict[str, Any]
|
||||
if not tenant_id:
|
||||
# The engine has no tenant context yet for orders created through
|
||||
# this API (orchestrator/main.py never threads tenant_id into
|
||||
# `config` — a pre-existing, separately-tracked gap). Rather than
|
||||
# fabricate a tenant to make the bridge call "work", skip it
|
||||
# honestly and say why.
|
||||
quality_bridge_metrics = {
|
||||
"status": "not_configured",
|
||||
"detail": "config.tenant_id missing — engine has no tenant context for this order",
|
||||
}
|
||||
else:
|
||||
bridge_result = await run_quality_bridge(
|
||||
tenant_id=tenant_id,
|
||||
project_id=config.get("project_id", "p_default"),
|
||||
dataset_id=f"0pt-order-{order_id}",
|
||||
rows=delivered,
|
||||
correlation_id=config.get("correlation_id"),
|
||||
)
|
||||
quality_bridge_metrics = bridge_result.as_metrics()
|
||||
if bridge_result.status == "ok":
|
||||
quality_info = {
|
||||
"score": bridge_result.score,
|
||||
"run_id": bridge_result.run_id,
|
||||
"run_status": bridge_result.run_status,
|
||||
"checked_by": "0pt-quality-bridge",
|
||||
}
|
||||
for r in delivered:
|
||||
for p in r.get("provenance", []):
|
||||
p["quality"] = quality_info
|
||||
elif bridge_result.status != "disabled":
|
||||
logger.warning(
|
||||
"0pt.quality_bridge did not complete for order %s: %s (%s)",
|
||||
order_id, bridge_result.status, bridge_result.detail,
|
||||
)
|
||||
|
||||
return {
|
||||
"stage": "qa", "pct": 100, "state": "done",
|
||||
"metrics": {
|
||||
"rows": len(delivered), "dropped_below_threshold": dropped,
|
||||
"per_market": per_market, "avg_confidence": avg_conf,
|
||||
"pipeline_version": PIPELINE_VERSION,
|
||||
"quality_bridge": quality_bridge_metrics,
|
||||
},
|
||||
"records": delivered,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
"""
|
||||
services/api/workers/quality_bridge.py
|
||||
=========================================
|
||||
P010: the TS/Python split-brain bridge (BEFORE_STATE.md §2). The production
|
||||
order pipeline (workers/pipeline.py) has always run its own one-line QA
|
||||
filter (`[r for r in deduped if r["confidence"] >= threshold]`) and never
|
||||
touched this repo's sophisticated, versioned TS QualityService (393 lines:
|
||||
rule registry, GE delegation, OpenLineage) — "noll bryggor mellan sig", per
|
||||
BEFORE_STATE.md.
|
||||
|
||||
This module is the ADDITIVE bridge, called optionally from pipeline.py. It
|
||||
is NOT a replacement for the Python filter and cannot become a silent one:
|
||||
|
||||
* Feature-flagged OFF by default (OPT_QUALITY_BRIDGE_ENABLED unset/"false").
|
||||
A disabled bridge makes zero network calls and returns immediately.
|
||||
* FAILS OPEN. Any failure mode (bridge URL unset, connection refused,
|
||||
timeout, non-2xx, malformed JSON) is caught here and reported back as
|
||||
`{"status": "unavailable", ...}` — it NEVER raises into pipeline.run_job,
|
||||
and it never changes which rows get delivered. `run_job` remains the
|
||||
single source of truth for delivery; this bridge is read-only/advisory.
|
||||
* ADVISORY ONLY, not a second gate — and this is a real limitation, not a
|
||||
choice made lightly: ExpectationEngine's execute() returns per-rule
|
||||
PASS/FAIL COUNTS and up to 20 unexpected-value SAMPLES
|
||||
(src/quality/expectation-engine.ts), not a per-row boolean array. There
|
||||
is no way, from a dataset-level score + a handful of samples, to know
|
||||
*which* of the N rows in this batch failed which rule — so per-row
|
||||
gating against the TS engine's result cannot be implemented honestly
|
||||
without first extending that engine's output contract (out of scope for
|
||||
this pass; flagged, not worked around with a guess). A future
|
||||
`OPT_QUALITY_BRIDGE_GATE_ENABLED` flag is deliberately NOT implemented
|
||||
here for that reason — see the docstring on `BridgeResult` below.
|
||||
|
||||
Transport: a direct HTTP call to the Next.js edge's
|
||||
POST /api/0pt/quality/bridge-run (app/api/0pt/quality/bridge-run/route.ts),
|
||||
reusing the same `X-Internal-Token` shared-secret convention lib/engine.ts
|
||||
already uses for the (opposite-direction) edge -> engine call, for
|
||||
consistency. This was chosen over an N0D job dispatch (the other option the
|
||||
brief raised) because: the TS QualityService instance the pipeline needs to
|
||||
reach lives in-process inside the Next.js app (src/services/quality.ts, an
|
||||
in-memory singleton, same pattern as the Source Registry before its Postgres
|
||||
persistence landed) — there is no durable store an N0D job could read/write
|
||||
through today, and dispatching a job whose only effect is "call this
|
||||
in-process singleton" would add a queue hop for no isolation benefit. A
|
||||
synchronous HTTP call, wrapped in a short timeout and a hard fail-open, is
|
||||
the simpler and more honest choice for a first cut; nothing here blocks a
|
||||
later move to N0D once the quality state has real (Postgres) persistence.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
_TIMEOUT = httpx.Timeout(4.0, connect=2.0) # short — this must never meaningfully slow delivery
|
||||
|
||||
|
||||
def _enabled() -> bool:
|
||||
return os.getenv("OPT_QUALITY_BRIDGE_ENABLED", "false").strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
def _bridge_url() -> str | None:
|
||||
base = os.getenv("OPT_QUALITY_BRIDGE_URL") or os.getenv("EDGE_URL")
|
||||
if not base:
|
||||
return None
|
||||
return f"{base.rstrip('/')}/api/0pt/quality/bridge-run"
|
||||
|
||||
|
||||
@dataclass
|
||||
class BridgeResult:
|
||||
"""
|
||||
Outcome of one bridge-run attempt. `status`:
|
||||
- "disabled": OPT_QUALITY_BRIDGE_ENABLED is not set — no call made.
|
||||
- "not_configured": enabled, but no OPT_QUALITY_BRIDGE_URL/EDGE_URL set.
|
||||
- "unavailable": enabled + configured, but the call failed (timeout,
|
||||
connection error, non-2xx, malformed body). Never raises.
|
||||
- "ok": a real run completed; `score`/`run_id`/`issue_count` are populated.
|
||||
|
||||
There is deliberately no `gated`/`dropped_rows` field — see the module
|
||||
docstring's "ADVISORY ONLY" section for why per-row gating isn't
|
||||
implementable against the current TS engine contract.
|
||||
"""
|
||||
status: str
|
||||
score: float | None = None
|
||||
run_id: str | None = None
|
||||
run_status: str | None = None
|
||||
issue_count: int | None = None
|
||||
detail: str | None = None
|
||||
|
||||
def as_metrics(self) -> dict[str, Any]:
|
||||
return {
|
||||
"status": self.status,
|
||||
"score": self.score,
|
||||
"run_id": self.run_id,
|
||||
"run_status": self.run_status,
|
||||
"issue_count": self.issue_count,
|
||||
**({"detail": self.detail} if self.detail else {}),
|
||||
}
|
||||
|
||||
|
||||
async def run_quality_bridge(
|
||||
*,
|
||||
tenant_id: str,
|
||||
project_id: str,
|
||||
dataset_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
correlation_id: str | None = None,
|
||||
) -> BridgeResult:
|
||||
"""Ask the TS QualityService to grade `rows`. Never raises — see module docstring."""
|
||||
if not _enabled():
|
||||
return BridgeResult(status="disabled")
|
||||
|
||||
url = _bridge_url()
|
||||
if not url:
|
||||
return BridgeResult(status="not_configured", detail="OPT_QUALITY_BRIDGE_URL/EDGE_URL not set")
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"tenant_id": tenant_id,
|
||||
"project_id": project_id,
|
||||
"dataset_id": dataset_id,
|
||||
"rows": rows,
|
||||
}
|
||||
if correlation_id:
|
||||
payload["correlation_id"] = correlation_id
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"X-Internal-Token": os.getenv("INTERNAL_TOKEN", ""),
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
r = await client.post(url, json=payload, headers=headers)
|
||||
if r.status_code not in (200, 201):
|
||||
return BridgeResult(status="unavailable", detail=f"HTTP {r.status_code}")
|
||||
body = r.json()
|
||||
run = body.get("run") or {}
|
||||
issues = body.get("issues") or []
|
||||
return BridgeResult(
|
||||
status="ok",
|
||||
score=run.get("score"),
|
||||
run_id=run.get("run_id"),
|
||||
run_status=run.get("status"),
|
||||
issue_count=len(issues),
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
return BridgeResult(status="unavailable", detail="timeout")
|
||||
except httpx.HTTPError as exc:
|
||||
return BridgeResult(status="unavailable", detail=f"{type(exc).__name__}: {exc}")
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
# Malformed/unexpected response body — treat exactly like network
|
||||
# unavailability, never propagate into the pipeline.
|
||||
return BridgeResult(status="unavailable", detail=f"malformed response: {type(exc).__name__}")
|
||||
@@ -2,6 +2,53 @@
|
||||
Entity resolution (docs/06): block, score, and link records across sources so the
|
||||
same company from GLEIF + Brreg + CVR collapses to one record with corroborated
|
||||
confidence. rapidfuzz only; deterministic; no external calls.
|
||||
|
||||
=======================================================================
|
||||
P065 — THIS IS THE SECOND ER ENGINE. IT IS NOT THE CANONICAL ONE.
|
||||
=======================================================================
|
||||
0pt has TWO entity-resolution implementations, and this is the simplified
|
||||
one:
|
||||
|
||||
* CANONICAL: src/er/entity-resolution-service.ts — the declared generic ER
|
||||
capability. Deterministic blocking + an explainable per-pair score,
|
||||
product-supplied HARD RULES that override the score in both directions,
|
||||
a configurable review band that hands ambiguous pairs to humans instead
|
||||
of auto-merging, and delegation of heavy probabilistic linkage (Splink)
|
||||
to 0pt-motor via the N0D job "0pt.er.batch_resolution".
|
||||
|
||||
* THIS FILE: blocking on (market, first initial) + a single
|
||||
rapidfuzz token_sort_ratio threshold, first-match-wins, auto-merge with
|
||||
no review band, no hard-rule hook, no explanation beyond the score.
|
||||
|
||||
P065's definition of done is that no second simplified production engine
|
||||
remains. It has NOT been met: this file is still what
|
||||
`workers/pipeline.py` calls (line ~110, `deduped = resolve(records)`).
|
||||
|
||||
WHY IT WAS NOT REPLACED IN THE P060-P068 PASS, stated plainly rather than
|
||||
quietly deferred: replacing it means turning this module into an ADAPTER
|
||||
that calls the canonical capability, and that change cannot be verified
|
||||
from this repo today —
|
||||
* `services/api` is not deployed by this repo's Dockerfile (which builds
|
||||
only the Next.js app), and api.0pt.app answers 404, so there is no
|
||||
running engine to test against;
|
||||
* no CI workflow runs services/api's pytest suite at all (verified:
|
||||
`grep -rn "pytest" .gitea/workflows .github/workflows` finds nothing),
|
||||
so a rewrite here would land with zero automated coverage.
|
||||
Rewriting it blind and calling P065 done would be exactly the
|
||||
"claimed complete, broken in operation" failure this programme exists to
|
||||
stop. The removal plan is recorded in STATUS.md.
|
||||
|
||||
Until then: do NOT add capability to this file. Any new ER behaviour
|
||||
belongs in the canonical TypeScript service.
|
||||
|
||||
Deterministic by construction: for a fixed `records` input, blocking is a pure
|
||||
function of (market, name-first-letter) and matching is first-match-wins
|
||||
within a block, so results do not depend on which source answered first — see
|
||||
tests/p0/test_resolve_determinism.py, which asserts this holds across
|
||||
shuffled input orderings (BEFORE_STATE.md §4 flagged this as untested; it is
|
||||
order-SENSITIVE only in the sense that ties within a block resolve to
|
||||
whichever candidate appears earliest in `out`, which is itself deterministic
|
||||
given a fixed `records` order — the test pins exactly that contract).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -19,9 +66,14 @@ def resolve(records: list[dict[str, Any]], threshold: int = 92) -> list[dict[str
|
||||
out: list[dict[str, Any]] = []
|
||||
for r in records:
|
||||
match = None
|
||||
match_score: float | None = None
|
||||
for o in out:
|
||||
if block_key(o) == block_key(r) and fuzz.token_sort_ratio(o.get("name", ""), r.get("name", "")) >= threshold:
|
||||
if block_key(o) != block_key(r):
|
||||
continue
|
||||
score = fuzz.token_sort_ratio(o.get("name", ""), r.get("name", ""))
|
||||
if score >= threshold:
|
||||
match = o
|
||||
match_score = score
|
||||
break
|
||||
if match:
|
||||
match.setdefault("merged_from", []).append(r.get("source"))
|
||||
@@ -44,6 +96,20 @@ def resolve(records: list[dict[str, Any]], threshold: int = 92) -> list[dict[str
|
||||
match["confidence"] = min(0.99, match.get("confidence", 0.0) + 0.1)
|
||||
match["agreed_sources"] = match.get("agreed_sources", 1) + 1
|
||||
seen_sources.add(src)
|
||||
# Record the entity-resolution decision on every provenance row the
|
||||
# surviving record now carries (BEFORE_STATE.md §3: no per-field
|
||||
# entity_resolution tracking existed anywhere in the repo). Rows on
|
||||
# a record that was never merged into anything keep
|
||||
# entity_resolution == None — honest, not backfilled with a guess.
|
||||
er_info = {
|
||||
"method": "blocking+fuzzy_token_sort_ratio",
|
||||
"block_key": list(block_key(match)),
|
||||
"match_score": match_score,
|
||||
"threshold": threshold,
|
||||
"merged_sources": sorted(s for s in match["_sources_seen"] if s is not None),
|
||||
}
|
||||
for p in existing_provenance:
|
||||
p["entity_resolution"] = er_info
|
||||
else:
|
||||
out.append({**r, "agreed_sources": 1, "_sources_seen": {r.get("source")} - {None}})
|
||||
for o in out:
|
||||
|
||||
+53
-16
@@ -1,19 +1,56 @@
|
||||
{
|
||||
"name": "0pt",
|
||||
"category": "platform",
|
||||
"subcategory": "data-os",
|
||||
"appId": "0PT",
|
||||
"owner": "SIAX Technology AB",
|
||||
"packageName": "@siax/0pt",
|
||||
"capabilityPrefix": "0pt",
|
||||
"deploy": {
|
||||
"target": "coolify",
|
||||
"node": null,
|
||||
"domain": "0pt.app",
|
||||
"healthPath": "/api/health"
|
||||
"$schema": "https://git.siax.io/sax3l/siax-standard/raw/branch/main/schema/siax.config.schema.json",
|
||||
"framework": "nextjs",
|
||||
"build": "pnpm run build",
|
||||
"start": "pnpm run start",
|
||||
"port": 3000,
|
||||
"healthcheck": {
|
||||
"path": "/api/health",
|
||||
"readyPath": "/api/ready"
|
||||
},
|
||||
"consumes": ["CL0UD", "ST0RE", "N0D", "AUD0", "INF0"],
|
||||
"replacesSaaS": ["Apollo", "Cognism", "ZoomInfo", "Clearbit"],
|
||||
"decoupled": false,
|
||||
"designSystem": null
|
||||
"environments": {
|
||||
"prod": {
|
||||
"domain": "0pt.app"
|
||||
}
|
||||
},
|
||||
"env": [
|
||||
"DATABASE_URL",
|
||||
"POSTGRES_URL",
|
||||
"POSTGRES_PRISMA_URL",
|
||||
"REDIS_URL",
|
||||
"AUTH_SECRET",
|
||||
"AUTH_URL",
|
||||
"DEV_LOGIN_KEY",
|
||||
"INTERNAL_TOKEN",
|
||||
"ZITADEL_ISSUER",
|
||||
"AUTH_ENFORCE",
|
||||
"ZITADEL_TENANT_CLAIM",
|
||||
"ENGINE_URL",
|
||||
"CL0UD_URL",
|
||||
"N0D_URL",
|
||||
"AUD0_URL",
|
||||
"INF0_URL",
|
||||
"ST0RE_URL",
|
||||
"STRIPE_SECRET_KEY",
|
||||
"STRIPE_WEBHOOK_SECRET",
|
||||
"NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY",
|
||||
"R2_ENDPOINT",
|
||||
"R2_BUCKET",
|
||||
"R2_ACCESS_KEY_ID",
|
||||
"R2_ACCOUNT_ID",
|
||||
"R2_SECRET_ACCESS_KEY",
|
||||
"DOC_INTELLIGENCE_ENDPOINT",
|
||||
"DOC_INTELLIGENCE_KEY",
|
||||
"DOC_INTELLIGENCE_API_VERSION",
|
||||
"GRANT_ENC_KEY",
|
||||
"ODOO_BASE_URL",
|
||||
"RESEND_API_KEY",
|
||||
"METABASE_SITE_URL",
|
||||
"METABASE_SECRET_KEY",
|
||||
"NOCODB_BASE_URL",
|
||||
"NOCODB_API_TOKEN",
|
||||
"NEXT_PUBLIC_APP_URL",
|
||||
"NEXT_PUBLIC_API_URL"
|
||||
],
|
||||
"decoupled": false
|
||||
}
|
||||
|
||||
+47
-22
@@ -1,34 +1,59 @@
|
||||
{
|
||||
"$schema": "https://docs.siax.io/schemas/siax.repo.schema.json",
|
||||
"name": "0pt",
|
||||
"$schema": "https://git.siax.io/sax3l/siax-standard/raw/branch/main/schema/siax.repo.schema.json",
|
||||
"repo": "0pt",
|
||||
"standardVersion": "2.0",
|
||||
"category": "platform",
|
||||
"subcategory": "data-os",
|
||||
"appId": "0PT",
|
||||
"capabilityPrefix": "0pt",
|
||||
"packageName": "@siax/0pt",
|
||||
"owner": "SIAX Technology AB",
|
||||
"codeowners": ["@sax3l"],
|
||||
"lifecycle": "active",
|
||||
"operationalState": "running",
|
||||
"gitHost": "git.siax.io",
|
||||
"node": null,
|
||||
"domains": [
|
||||
"0pt.app",
|
||||
"api.0pt.app"
|
||||
{
|
||||
"fqdn": "0pt.app",
|
||||
"env": "prod",
|
||||
"purpose": "Next.js edge plane -- marketing, catalog, order flow, customer/admin portal, capability API",
|
||||
"registrar": "EJ VERIFIERAT fran detta repo -- se docs/architecture/ARCHITECTURE.md SS6/SS9 (BACKLOG P0-15)"
|
||||
},
|
||||
{
|
||||
"fqdn": "api.0pt.app",
|
||||
"env": "prod",
|
||||
"purpose": "Target API host (Caddy) -- documented target, not verified live (contracts/openapi.yaml server list)",
|
||||
"registrar": "EJ VERIFIERAT fran detta repo -- se docs/architecture/ARCHITECTURE.md SS6/SS9 (BACKLOG P0-15)"
|
||||
}
|
||||
],
|
||||
"deploy": {
|
||||
"target": "coolify",
|
||||
"healthPath": "/api/health"
|
||||
"consumes": ["CL0UD", "ST0RE", "N0D", "AUD0", "INF0"],
|
||||
"consumedBy": [],
|
||||
"children": [
|
||||
{
|
||||
"path": "app",
|
||||
"purpose": "Next.js edge plane -- UI + capability/order/payment API route handlers",
|
||||
"lifecycle": "active"
|
||||
},
|
||||
{
|
||||
"path": "services/api",
|
||||
"purpose": "FastAPI data engine (orchestrator + workers) -- legal sourcing chain, estimates/orders/screening",
|
||||
"lifecycle": "active"
|
||||
},
|
||||
{
|
||||
"path": "services/scrapers",
|
||||
"purpose": "Scraper stack including WAF-bypass code -- kept per a dated, attributed account-owner decision under active review; NOT part of the compliant sourcing chain (see SECURITY.md, CLAUDE.md SS1)",
|
||||
"lifecycle": "active"
|
||||
},
|
||||
{
|
||||
"path": "packages/schemas",
|
||||
"purpose": "Shared JSON Schema definitions for 0pt data contracts (dataset/iceberg/lineage/signal/rate-policy)",
|
||||
"lifecycle": "active"
|
||||
}
|
||||
],
|
||||
"replacesSaaS": ["Apollo", "Cognism", "ZoomInfo", "Clearbit"],
|
||||
"contracts": {
|
||||
"openapi": "contracts/openapi.yaml"
|
||||
},
|
||||
"consumes": [
|
||||
"CL0UD",
|
||||
"ST0RE",
|
||||
"N0D",
|
||||
"AUD0",
|
||||
"INF0"
|
||||
],
|
||||
"replacesSaaS": [
|
||||
"Apollo",
|
||||
"Cognism",
|
||||
"ZoomInfo",
|
||||
"Clearbit"
|
||||
],
|
||||
"decoupled": false,
|
||||
"designSystem": null
|
||||
"decoupled": false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
{
|
||||
"$schema": "https://git.siax.io/sax3l/siax-standard/raw/branch/main/schema/siax.repo.v3.schema.json",
|
||||
"standardVersion": "3.0",
|
||||
"repo": "0PT",
|
||||
"tier": "core-platform",
|
||||
"category": "platform",
|
||||
"subcategory": "data-os",
|
||||
"appId": "0PT",
|
||||
"capabilityPrefix": "0pt",
|
||||
"packageName": "@siax/0pt",
|
||||
"owner": "SIAX Technology AB",
|
||||
"lifecycle": "active",
|
||||
"operationalState": "degraded",
|
||||
"gitHost": "git.siax.io",
|
||||
"node": null,
|
||||
"domains": [
|
||||
{
|
||||
"fqdn": "0pt.app",
|
||||
"env": "prod",
|
||||
"purpose": "Next.js edge plane (marketing, catalog, order flow, customer/admin portal, capability API). Live on Vercel, NOT on the SIAX fleet (docs/runbook/deploy.md). Probe 2026-09-04: GET /api/health 200 with db=not_configured, queue=not_configured; GET /api/ready 404 -- the deployed build predates main's readiness route."
|
||||
},
|
||||
{
|
||||
"fqdn": "api.0pt.app",
|
||||
"env": "prod",
|
||||
"purpose": "Documented target host for the FastAPI data engine (contracts/openapi.yaml servers list). NOT live: docs/slo.md says the engine is not deployed; probe 2026-09-04 returned HTTP 404."
|
||||
}
|
||||
],
|
||||
"consumes": ["siax-standard", "cl0ud", "st0re", "n0d", "aud0", "inf0"],
|
||||
"consumedBy": [],
|
||||
"replacesSaaS": ["Apollo", "Cognism", "ZoomInfo", "Clearbit"],
|
||||
"children": [
|
||||
{
|
||||
"path": "app",
|
||||
"purpose": "Next.js edge plane -- UI + capability/order/payment API route handlers",
|
||||
"lifecycle": "active"
|
||||
},
|
||||
{
|
||||
"path": "services/api",
|
||||
"purpose": "FastAPI data engine (orchestrator + workers) -- legal sourcing chain, estimates/orders/screening. Python tests not wired into CI (PRODUCTION_EVIDENCE.json)",
|
||||
"lifecycle": "active"
|
||||
},
|
||||
{
|
||||
"path": "services/scrapers",
|
||||
"purpose": "Scraper stack including WAF-bypass code -- kept per a dated, attributed account-owner decision under active review; deletion PR open, NOT merged; NOT part of the compliant sourcing chain (SECURITY.md, CLAUDE.md section 1)",
|
||||
"lifecycle": "active"
|
||||
},
|
||||
{
|
||||
"path": "packages/schemas",
|
||||
"purpose": "Shared JSON Schema definitions for 0pt data contracts (dataset/iceberg/lineage/signal/rate-policy); consumed by path, not an SDK",
|
||||
"lifecycle": "active"
|
||||
}
|
||||
],
|
||||
"runtime": {
|
||||
"language": "typescript",
|
||||
"node": ">=22",
|
||||
"framework": "nextjs",
|
||||
"packageManager": "pnpm"
|
||||
},
|
||||
"canonical": true,
|
||||
"parentRepo": null,
|
||||
"capabilities": [
|
||||
"prospekt.email_verify",
|
||||
"prospekt.jobs_write",
|
||||
"prospekt.jobs_read",
|
||||
"prospekt.search_read",
|
||||
"prospekt.search_write",
|
||||
"0pt.query_source",
|
||||
"0pt.document_ocr",
|
||||
"0pt.write_external_record",
|
||||
"0pt.lakehouse_mapping_manage",
|
||||
"0pt.lakehouse_write",
|
||||
"0pt.source_rate_policy_manage",
|
||||
"0pt.screen_sanctions",
|
||||
"0pt.estimate_compute",
|
||||
"0pt.vehicle_lookup",
|
||||
"0pt.catalog_list",
|
||||
"0pt.source_registry_list",
|
||||
"0pt.order_create",
|
||||
"0pt.order_payment",
|
||||
"0pt.write_iceberg_table"
|
||||
],
|
||||
"surfaces": ["api", "ui"],
|
||||
"contracts": {
|
||||
"openapi": "contracts/openapi.yaml"
|
||||
},
|
||||
"identity": {
|
||||
"workloadIdentity": false,
|
||||
"authProvider": "cl0ud",
|
||||
"tenantScoped": true
|
||||
},
|
||||
"persistence": {
|
||||
"stateful": true,
|
||||
"database": "postgres",
|
||||
"storage": "st0re"
|
||||
},
|
||||
"deployment": {
|
||||
"node": null,
|
||||
"dedicated": false,
|
||||
"environments": ["prod"]
|
||||
},
|
||||
"observability": {
|
||||
"health": true,
|
||||
"ready": true,
|
||||
"metrics": false,
|
||||
"otel": false
|
||||
},
|
||||
"reliability": {
|
||||
"backupFrequency": "none",
|
||||
"provenRestore": false
|
||||
},
|
||||
"engines": ["postgres", "opensearch", "clickhouse"],
|
||||
"licenses": {
|
||||
"spdx": "UNLICENSED",
|
||||
"approved": false
|
||||
},
|
||||
"productionEvidence": {
|
||||
"verified": false
|
||||
},
|
||||
"decoupled": false
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
-- 0pt Source Registry persistence (Master-DoD §10; BEFORE_STATE.md §1).
|
||||
--
|
||||
-- Backs src/services/source-service.ts (0pt.source.schema.v1, the base
|
||||
-- registry) and src/services/source-registry-service.ts (rights/license/
|
||||
-- purpose, provenance, health, freshness — the P0 Source Registry surface
|
||||
-- layered on top). Both previously held their state in bare `new Map()`s,
|
||||
-- scoped per service *instance* (not even shared within one process) and
|
||||
-- hard-blocked from production boot by resolveStoreMode
|
||||
-- (src/platform/dependency-mode.ts) for lack of a DB. This migration is the
|
||||
-- DB half of closing that gap; src/repositories/* is the code half.
|
||||
--
|
||||
-- Deliberately numbered 002, not 006: sql/004_iceberg_write_boundary.sql's
|
||||
-- opt_source_rate_policies table already carries
|
||||
-- `source_id TEXT NOT NULL REFERENCES opt_sources (source_id)` — a
|
||||
-- forward reference to a table that has never existed in any sql/ file
|
||||
-- (opt_sources was, until this file, only ever a doc-comment aspiration —
|
||||
-- see the "What Postgres migration would require" section of
|
||||
-- BEFORE_STATE.md §1 and the retracted `sql/002_p0_dataset_quality.sql`
|
||||
-- claim in CLAUDE.md/app/api/0pt/sources/[sourceId]/query/route.ts, an
|
||||
-- UNRELATED fabricated claim about a different, never-written file that
|
||||
-- happened to reuse this same number). Applying 004 before opt_sources
|
||||
-- exists would fail outright on a fresh database; this file must run
|
||||
-- between 001 and 004 to make 004's existing FK resolvable. This is a
|
||||
-- pre-existing ordering bug in 004, not introduced here — fixed here only
|
||||
-- because this pass needed opt_sources to exist anyway.
|
||||
|
||||
-- ── 0pt.source.schema.v1 (src/types/source.ts) ─────────────────────────
|
||||
CREATE TABLE IF NOT EXISTS opt_sources (
|
||||
source_id TEXT PRIMARY KEY,
|
||||
tenant_id TEXT NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
source_type TEXT NOT NULL, -- database | file | web | api | stream | manual
|
||||
name TEXT NOT NULL,
|
||||
adapter_id TEXT, -- e.g. "fabric_onelake_sql" (src/adapters/fabric-onelake-adapter.ts)
|
||||
access TEXT, -- api | bulk | authorized_web | licensed (never a bypass mode)
|
||||
description TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'active', -- active | paused | archived | deprecated
|
||||
correlation_id TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ,
|
||||
created_by TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS opt_sources_tenant_project_idx ON opt_sources (tenant_id, project_id);
|
||||
CREATE INDEX IF NOT EXISTS opt_sources_type_idx ON opt_sources (source_type);
|
||||
CREATE INDEX IF NOT EXISTS opt_sources_status_idx ON opt_sources (status);
|
||||
|
||||
-- ── Rights / license / purpose metadata (src/types/source-registry.ts) ─
|
||||
-- One row per source (SourceRegistryService.setRights is an upsert-by-
|
||||
-- source_id, same as the Map it replaces). No rights row = fail-closed
|
||||
-- deny in assertPurposeAllowed — that logic lives in the service, not here.
|
||||
CREATE TABLE IF NOT EXISTS opt_source_rights (
|
||||
source_id TEXT PRIMARY KEY REFERENCES opt_sources (source_id),
|
||||
tenant_id TEXT NOT NULL,
|
||||
license_id TEXT NOT NULL,
|
||||
license_url TEXT,
|
||||
allowed_purposes TEXT[] NOT NULL DEFAULT '{}',
|
||||
pii_allowed BOOLEAN NOT NULL DEFAULT false,
|
||||
redistribution TEXT NOT NULL, -- internal | customer_deliverable | public | prohibited
|
||||
attribution TEXT,
|
||||
verified_at TEXT NOT NULL, -- ISO date the assessment was made (date-only in practice; kept as TEXT, not TIMESTAMPTZ, so it round-trips byte-for-byte)
|
||||
verified_by TEXT NOT NULL,
|
||||
notes TEXT,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS opt_source_rights_tenant_idx ON opt_source_rights (tenant_id);
|
||||
|
||||
-- ── Provenance / method metadata (per-SOURCE, not per-field — see
|
||||
-- BEFORE_STATE.md §3 for the distinction from bridge_provenance) ────────
|
||||
CREATE TABLE IF NOT EXISTS opt_source_provenance (
|
||||
source_id TEXT PRIMARY KEY REFERENCES opt_sources (source_id),
|
||||
tenant_id TEXT NOT NULL,
|
||||
method TEXT NOT NULL, -- official_api | bulk_download | authorized_web | licensed_feed | manual_upload
|
||||
authority TEXT,
|
||||
compliance_evidence_ref TEXT,
|
||||
cadence TEXT,
|
||||
freshness_sla_hours NUMERIC,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS opt_source_provenance_tenant_idx ON opt_source_provenance (tenant_id);
|
||||
|
||||
-- ── Health observations (source.health) — an operational signal, NOT an
|
||||
-- audit log (AUD0 owns history). Bounded to the latest 100 rows per
|
||||
-- source_id by the repository layer (src/repositories/postgres-source-
|
||||
-- registry-repository.ts), mirroring the in-memory implementation's
|
||||
-- `list.splice(0, list.length - 100)` bound.
|
||||
CREATE TABLE IF NOT EXISTS opt_source_health_checks (
|
||||
health_check_id BIGSERIAL PRIMARY KEY,
|
||||
source_id TEXT NOT NULL REFERENCES opt_sources (source_id),
|
||||
tenant_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL, -- up | degraded | down | unknown
|
||||
checked_at TIMESTAMPTZ NOT NULL,
|
||||
latency_ms INT,
|
||||
detail TEXT -- short operator-facing text; never a raw upstream body
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS opt_source_health_checks_source_idx ON opt_source_health_checks (source_id, checked_at DESC, health_check_id DESC);
|
||||
|
||||
-- ── Freshness: last successful observation per source (source.health's
|
||||
-- "up" checks call recordSuccess; getFreshness reads this + provenance's
|
||||
-- freshness_sla_hours). One row per source — upserted, not appended.
|
||||
CREATE TABLE IF NOT EXISTS opt_source_freshness (
|
||||
source_id TEXT PRIMARY KEY REFERENCES opt_sources (source_id),
|
||||
tenant_id TEXT NOT NULL,
|
||||
last_success_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS opt_source_freshness_tenant_idx ON opt_source_freshness (tenant_id);
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user