0pt/docs/03-data-model.md

03 · Data model

Postgres is the backbone. Two halves: data entities (what we sell) and platform tables (orders, users, jobs). Every fact links to bridge_provenance. Migrations via Alembic (Python) or sql/ migrations checked into the repo — pick one and be consistent; align with existing src/workers/db.py.

Conventions

• snake_case, UTC timestamptz, soft-delete via deleted_at where useful.
• Natural keys for registries (org_nr), surrogate uuid PKs for platform tables.
• created_at / updated_at on every table.
• All monetary amounts in öre (integer) to avoid float errors; currency SEK default.

Data entities

-- A company (the core registry entity)
CREATE TABLE dim_company (
  org_nr        text PRIMARY KEY,            -- national reg. number (per market format)
  market        text NOT NULL,               -- ISO country code: SE/NO/DK/FI/...
  lei           text,                         -- GLEIF cross-border key
  name          text NOT NULL,
  status        text,                         -- active/dissolved/...
  nace          text,                         -- NACE/SNI code
  registered_at date,
  address       jsonb,
  website       text,
  updated_at    timestamptz NOT NULL DEFAULT now(),
  created_at    timestamptz NOT NULL DEFAULT now()
);

CREATE TABLE fact_financials (
  org_nr      text REFERENCES dim_company(org_nr),
  fiscal_year int  NOT NULL,
  revenue     bigint,                          -- öre
  result      bigint,
  employees   int,
  source_id   text REFERENCES dim_source(source_id),
  PRIMARY KEY (org_nr, fiscal_year)
);

CREATE TABLE dim_contact (
  contact_id  uuid PRIMARY KEY DEFAULT gen_random_uuid(),
  org_nr      text REFERENCES dim_company(org_nr),
  name        text,
  role        text,
  email       text,
  phone       text,
  linkedin    text,
  -- PII: subject to GDPR; lawful basis recorded per delivery
  updated_at  timestamptz NOT NULL DEFAULT now()
);

CREATE TABLE dim_ubo (                          -- ultimate beneficial owner
  ubo_id   uuid PRIMARY KEY DEFAULT gen_random_uuid(),
  org_nr   text REFERENCES dim_company(org_nr),
  owner    text,
  share    numeric(5,2)
);

CREATE TABLE dim_vehicle (                       -- VehIQ synergy
  vin         text PRIMARY KEY,
  market      text NOT NULL,
  reg_plate   text,
  make        text, model text, year int,
  owner_org   text REFERENCES dim_company(org_nr),
  specs       jsonb,
  updated_at  timestamptz NOT NULL DEFAULT now()
);

Provenance model (the signature — every fact links here)

CREATE TABLE dim_source (
  source_id   text PRIMARY KEY,                 -- 'gleif','brreg','companies_house',...
  name        text NOT NULL,
  market      text,                              -- or 'global'
  access      text,                              -- 'api'|'bulk'|'authorized_web'|'licensed'
  licence     text,                              -- short licence note
  cadence     text,                              -- 'daily'|'weekly'|'annual'|...
  url         text
);

-- One row per (entity field) we deliver, linking value → where it came from + confidence
CREATE TABLE bridge_provenance (
  field_id    uuid PRIMARY KEY DEFAULT gen_random_uuid(),
  entity_type text NOT NULL,                     -- 'company'|'contact'|'vehicle'|'financials'
  entity_key  text NOT NULL,                     -- org_nr / vin / contact_id
  field_name  text NOT NULL,                     -- 'revenue','email',...
  source_id   text REFERENCES dim_source(source_id),
  freshness   date,                              -- when this value was confirmed
  confidence  numeric(4,3) NOT NULL,             -- 0.000–1.000
  level       smallint NOT NULL,                 -- 0 unavailable/licensed, 1 single, 2 partial, 3 corroborated
  agreed_sources int DEFAULT 1,                  -- # independent sources that agree
  UNIQUE (entity_type, entity_key, field_name)
);
CREATE INDEX ON bridge_provenance (entity_type, entity_key);

Rule: no value ships without a bridge_provenance row. Level 0 = delivered as "not available / licensed", never a guessed value.

Platform tables

CREATE TABLE orgs (
  org_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
  name text NOT NULL, created_at timestamptz DEFAULT now()
);
CREATE TABLE users (
  user_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
  org_id uuid REFERENCES orgs(org_id),
  email text UNIQUE NOT NULL,
  role text NOT NULL DEFAULT 'member',           -- 'owner'|'admin'|'member'|'siax_admin'
  created_at timestamptz DEFAULT now()
);

CREATE TABLE orders (
  order_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
  org_id uuid REFERENCES orgs(org_id),
  flow text NOT NULL,                             -- 'catalog'|'custom'|'subscription'
  status text NOT NULL DEFAULT 'draft',           -- see lifecycle in docs/01
  config jsonb NOT NULL,                          -- configurator state (markets, fields, ...)
  total_ore bigint, deposit_ore bigint, balance_ore bigint,
  est_days int, est_confidence numeric(4,3),
  created_at timestamptz DEFAULT now(), updated_at timestamptz DEFAULT now()
);

CREATE TABLE order_items (                         -- resolved line items / dataset refs
  item_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
  order_id uuid REFERENCES orders(order_id),
  kind text, ref text, qty int, meta jsonb
);

CREATE TABLE jobs (                                -- one production run per order
  job_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
  order_id uuid REFERENCES orders(order_id),
  stage text,                                      -- sourcing|extraction|structuring|enrichment|qa
  pct numeric(5,2) DEFAULT 0,
  state text DEFAULT 'queued',                     -- queued|running|done|failed
  metrics jsonb,                                   -- rows, confidence, per-market progress
  started_at timestamptz, finished_at timestamptz
);

CREATE TABLE deliveries (
  delivery_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
  order_id uuid REFERENCES orders(order_id),
  formats text[], destinations text[],
  record_count int, avg_confidence numeric(4,3),
  quality_report jsonb, provenance_export_url text,
  files jsonb,                                     -- [{format,url,bytes}]
  created_at timestamptz DEFAULT now()
);

CREATE TABLE subscriptions (
  sub_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
  order_id uuid REFERENCES orders(order_id),
  cadence text,                                    -- weekly|daily
  webhook_url text, status text DEFAULT 'active',
  next_run_at timestamptz
);

CREATE TABLE files (                               -- customer uploads
  file_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
  org_id uuid REFERENCES orgs(org_id),
  order_id uuid REFERENCES orders(order_id),
  name text, mime text, bytes bigint, r2_key text,
  created_at timestamptz DEFAULT now()
);

CREATE TABLE db_grants (                           -- customer read-only DB access (encrypted)
  grant_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
  org_id uuid REFERENCES orgs(org_id),
  order_id uuid REFERENCES orders(order_id),
  conn_encrypted bytea NOT NULL,                   -- encrypted connection string
  scope text, expires_at timestamptz, revoked_at timestamptz,
  created_at timestamptz DEFAULT now()
);

CREATE TABLE audit_log (
  id bigserial PRIMARY KEY,
  actor text, action text, target text, meta jsonb,
  at timestamptz DEFAULT now()
);

CREATE TABLE payments (
  payment_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
  order_id uuid REFERENCES orders(order_id),
  kind text,                                       -- 'deposit'|'balance'
  amount_ore bigint, stripe_ref text,
  status text,                                     -- 'pending'|'captured'|'released'|'failed'
  created_at timestamptz DEFAULT now()
);

pgvector

CREATE EXTENSION vector; Use for entity matching/dedup and for the AI-assist (semantic match of NL request → catalog/config). Store embeddings in a embeddings table keyed by (entity_type, entity_key) or alongside catalog rows.

Row-level security & catalog

• Catalog products: a catalog table (or view) over curated datasets with sample rows, counts, coverage, price.
• Per-customer data isolation enforced by org_id filtering in the API and (where exposed via NocoDB/Metabase) by scoped views/connection params, plus Postgres RLS where practical.