fix(recovery): restore source removed by production documentation merge
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
# `analytics/dbt/` — BI-P022
|
||||
|
||||
The canonical SIAX transformation project. **Nothing in this repository runs
|
||||
it.** Execution lives in N0D (`n0d#38`), against the contract in
|
||||
`src/analytics/transform/dbt-contract.ts`.
|
||||
|
||||
## Layers, and why they map onto BI-P003
|
||||
|
||||
| directory | tier (BI-P003) | may read | must not contain |
|
||||
| ---------------------- | -------------- | --------------------- | --------------------- |
|
||||
| `models/sources/` | `RAW` | — (declarations only) | any SQL |
|
||||
| `models/staging/` | `NORMALIZED` | sources | joins, business logic |
|
||||
| `models/intermediate/` | `CURATED` | staging | presentation shaping |
|
||||
| `models/marts/` | `SERVING` | intermediate | direct source reads |
|
||||
|
||||
`assertLayerDependenciesLegal` enforces the "may read" column against the
|
||||
dependency graph in `manifest.json`. A mart reading a source directly is
|
||||
refused — that is raw data reaching a dashboard, which is precisely what
|
||||
BI-P003's tier rule exists to prevent, and a convention nobody checks would not
|
||||
prevent it.
|
||||
|
||||
## What a run must produce
|
||||
|
||||
`manifest.json` and `run_results.json`, both registered as ST0RE artifacts.
|
||||
`assertValidDbtJobResult` refuses a result missing either: without them the run
|
||||
cannot be audited and its tests cannot be ingested by BI-P025's
|
||||
`DbtQualityProvider`.
|
||||
|
||||
It also refuses:
|
||||
|
||||
- a result whose revision differs from the request — the run is not reproducible;
|
||||
- a result whose observed dbt version differs from the pin — a runtime that is
|
||||
not the pinned one silently changes what the models mean;
|
||||
- `status: success` with **zero nodes executed** — a dbt selector that matches
|
||||
nothing exits 0, which is the dbt-shaped false green.
|
||||
|
||||
## Version posture
|
||||
|
||||
`DBT_RUNTIME_PIN` records dbt Core 1.10.13 (stable line) with
|
||||
`dbt-clickhouse` 1.9.2. **Both review flags are `false`.** No licence or
|
||||
supply-chain review has been performed in this repo, and none is claimed.
|
||||
`assertDbtRuntimeCertified` refuses a `production` run until they are true, so
|
||||
BI-P022's "after current license/security verification" is a gate rather than a
|
||||
sentence in a document.
|
||||
|
||||
## Credentials
|
||||
|
||||
A dbt run never receives a tenant's warehouse credential from this repo. N0D
|
||||
resolves a credential whose write scope matches `target_layer` and
|
||||
`expected_target_schema`, and the contract refuses a request whose expected
|
||||
schema is not scoped to the job's tenant.
|
||||
@@ -0,0 +1,43 @@
|
||||
# BI-P022 — the canonical SIAX analytics transformation project.
|
||||
#
|
||||
# Execution happens in N0D (n0d#38), never here. This project is the MODELS and
|
||||
# the CONVENTIONS; src/analytics/transform/dbt-contract.ts is the job contract
|
||||
# N0D consumes, and it is what refuses a run that is not reproducible.
|
||||
name: siax_analytics
|
||||
version: 1.0.0
|
||||
config-version: 2
|
||||
profile: siax_analytics
|
||||
|
||||
model-paths: ['models']
|
||||
test-paths: ['tests']
|
||||
snapshot-paths: ['snapshots']
|
||||
macro-paths: ['macros']
|
||||
|
||||
# The four layers map onto BI-P003's tiers rather than inventing a fifth
|
||||
# vocabulary. `+tags` carries the tier so a manifest reader can assign one
|
||||
# without parsing paths, and dbt-contract.ts asserts the mapping.
|
||||
models:
|
||||
siax_analytics:
|
||||
+materialized: view
|
||||
staging:
|
||||
+schema: staging
|
||||
+materialized: view
|
||||
+tags: ['tier:NORMALIZED']
|
||||
intermediate:
|
||||
+schema: intermediate
|
||||
+materialized: table
|
||||
+tags: ['tier:CURATED']
|
||||
marts:
|
||||
+schema: marts
|
||||
+materialized: table
|
||||
+tags: ['tier:SERVING']
|
||||
|
||||
snapshots:
|
||||
siax_analytics:
|
||||
+target_schema: snapshots
|
||||
|
||||
# `on_schema_change: fail` rather than `sync_all_columns`: a silently widened
|
||||
# table is how a downstream metric starts averaging a column that did not exist
|
||||
# yesterday. BI-P004's contract gate is the same rule one layer up.
|
||||
vars:
|
||||
siax_tenant_id: '__set_by_n0d__'
|
||||
@@ -0,0 +1,16 @@
|
||||
-- CURATED (BI-P003). Joins and conformed entities. Reads STAGING only.
|
||||
{{ config(materialized='table', tags=['tier:CURATED']) }}
|
||||
|
||||
select
|
||||
e.event_id,
|
||||
e.tenant_id,
|
||||
e.customer_id,
|
||||
e.amount_sek,
|
||||
e.channel,
|
||||
coalesce(c.country_code, e.country_code) as country_code,
|
||||
c.segment,
|
||||
e.invoiced_at
|
||||
from {{ ref('stg_sales_events') }} e
|
||||
left join {{ ref('stg_customers') }} c
|
||||
on e.customer_id = c.customer_id
|
||||
and e.tenant_id = c.tenant_id -- the join itself stays tenant-scoped
|
||||
@@ -0,0 +1,17 @@
|
||||
-- SERVING (BI-P003). What a dashboard or a BI-P028 metric reads.
|
||||
--
|
||||
-- Reads INTERMEDIATE only. Reading a source here would skip two layers, and
|
||||
-- assertLayerDependenciesLegal refuses that against the manifest graph.
|
||||
{{ config(materialized='table', tags=['tier:SERVING']) }}
|
||||
|
||||
select
|
||||
tenant_id,
|
||||
date_trunc('day', invoiced_at) as day,
|
||||
country_code,
|
||||
channel,
|
||||
segment,
|
||||
count(*) as order_count,
|
||||
sum(amount_sek) as net_revenue_sek
|
||||
from {{ ref('int_sales_enriched') }}
|
||||
where channel != 'internal' -- matches the net_revenue metric's baked filter
|
||||
group by 1, 2, 3, 4, 5
|
||||
@@ -0,0 +1,19 @@
|
||||
version: 2
|
||||
|
||||
models:
|
||||
- name: mart_sales_daily
|
||||
description: >
|
||||
SERVING projection behind the `net_revenue` and `orders` metrics.
|
||||
A blocking test failure here quarantines the serving projection
|
||||
(BI-P025); the quarantine is enforced on the READ side.
|
||||
columns:
|
||||
- name: tenant_id
|
||||
tests: [not_null]
|
||||
- name: day
|
||||
tests: [not_null]
|
||||
- name: net_revenue_sek
|
||||
tests: [not_null]
|
||||
- name: order_count
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_utils.accepted_range: { min_value: 1, inclusive: true }
|
||||
@@ -0,0 +1,34 @@
|
||||
version: 2
|
||||
|
||||
# RAW. Declared, never modelled. A source has no SQL of its own — anything that
|
||||
# transforms belongs in staging, where BI-P003 assigns it NORMALIZED.
|
||||
sources:
|
||||
- name: opt_raw
|
||||
description: >
|
||||
Landed, untransformed data as delivered by the 0pt sourcing engine.
|
||||
RAW tier: no BI surface may read this directly without a recorded
|
||||
BI-P003 exception.
|
||||
schema: "{{ var('siax_tenant_id') }}_raw"
|
||||
loaded_at_field: ingested_at
|
||||
freshness:
|
||||
# BI-P026's DELAYED/STALE thresholds, expressed where dbt can act on them.
|
||||
warn_after: { count: 90, period: minute }
|
||||
error_after: { count: 180, period: minute }
|
||||
tables:
|
||||
- name: sales_events
|
||||
description: One row per invoiced sales line, as landed.
|
||||
columns:
|
||||
- name: event_id
|
||||
tests: [not_null, unique]
|
||||
- name: tenant_id
|
||||
description: Present on every raw row; the tier boundary depends on it.
|
||||
tests: [not_null]
|
||||
- name: ingested_at
|
||||
tests: [not_null]
|
||||
- name: customers
|
||||
description: Customer master as landed.
|
||||
columns:
|
||||
- name: customer_id
|
||||
tests: [not_null, unique]
|
||||
- name: tenant_id
|
||||
tests: [not_null]
|
||||
@@ -0,0 +1,26 @@
|
||||
version: 2
|
||||
|
||||
models:
|
||||
- name: stg_sales_events
|
||||
description: Normalized sales events — one row per event_id, latest landing wins.
|
||||
columns:
|
||||
- name: event_id
|
||||
tests: [not_null, unique]
|
||||
- name: tenant_id
|
||||
tests: [not_null]
|
||||
- name: amount_sek
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_utils.accepted_range:
|
||||
min_value: 0
|
||||
inclusive: true
|
||||
- name: channel
|
||||
tests:
|
||||
- accepted_values:
|
||||
values: ['web', 'retail', 'partner', 'internal']
|
||||
- name: stg_customers
|
||||
columns:
|
||||
- name: customer_id
|
||||
tests: [not_null, unique]
|
||||
- name: tenant_id
|
||||
tests: [not_null]
|
||||
@@ -0,0 +1,11 @@
|
||||
-- NORMALIZED (BI-P003).
|
||||
{{ config(materialized='view', tags=['tier:NORMALIZED']) }}
|
||||
|
||||
select
|
||||
cast(customer_id as varchar) as customer_id,
|
||||
cast(tenant_id as varchar) as tenant_id,
|
||||
cast(name as varchar) as customer_name,
|
||||
cast(segment as varchar) as segment,
|
||||
cast(country as varchar) as country_code
|
||||
from {{ source('opt_raw', 'customers') }}
|
||||
where tenant_id = '{{ var("siax_tenant_id") }}'
|
||||
@@ -0,0 +1,30 @@
|
||||
-- NORMALIZED (BI-P003). Rename, cast, deduplicate. No joins, no business logic.
|
||||
--
|
||||
-- The tenant predicate is applied HERE, at the first modelled layer, so every
|
||||
-- downstream model inherits it and no later model has to remember. It is the
|
||||
-- same forced-predicate rule as the query compiler's, one layer up.
|
||||
{{ config(materialized='view', tags=['tier:NORMALIZED']) }}
|
||||
|
||||
with source as (
|
||||
select * from {{ source('opt_raw', 'sales_events') }}
|
||||
where tenant_id = '{{ var("siax_tenant_id") }}'
|
||||
),
|
||||
|
||||
deduplicated as (
|
||||
select
|
||||
*,
|
||||
row_number() over (partition by event_id order by ingested_at desc) as _rn
|
||||
from source
|
||||
)
|
||||
|
||||
select
|
||||
cast(event_id as varchar) as event_id,
|
||||
cast(tenant_id as varchar) as tenant_id,
|
||||
cast(customer_id as varchar) as customer_id,
|
||||
cast(amount as double) as amount_sek,
|
||||
cast(channel as varchar) as channel,
|
||||
cast(country as varchar) as country_code,
|
||||
cast(invoiced_at as timestamp) as invoiced_at,
|
||||
cast(ingested_at as timestamp) as ingested_at
|
||||
from deduplicated
|
||||
where _rn = 1
|
||||
@@ -0,0 +1,15 @@
|
||||
-- Slowly-changing customer attributes, so a historical metric can be
|
||||
-- recomputed against what a customer's segment WAS, not what it is now.
|
||||
{% snapshot snap_customers %}
|
||||
{{
|
||||
config(
|
||||
target_schema='snapshots',
|
||||
unique_key='customer_id',
|
||||
strategy='check',
|
||||
check_cols=['segment', 'country_code'],
|
||||
invalidate_hard_deletes=True
|
||||
)
|
||||
}}
|
||||
select customer_id, tenant_id, customer_name, segment, country_code
|
||||
from {{ ref('stg_customers') }}
|
||||
{% endsnapshot %}
|
||||
@@ -0,0 +1,7 @@
|
||||
-- Fails when the serving mart is EMPTY.
|
||||
--
|
||||
-- Without this, every column test passes trivially over zero rows and the
|
||||
-- whole suite reports green while the pipeline is dead — the same false green
|
||||
-- BI-P025's NativeQualityProvider refuses.
|
||||
select 1 as empty_mart
|
||||
having (select count(*) from {{ ref('mart_sales_daily') }}) = 0
|
||||
@@ -0,0 +1,11 @@
|
||||
-- A singular test that FAILS if the serving mart ever contains a row belonging
|
||||
-- to a tenant other than the one the run was scoped to.
|
||||
--
|
||||
-- The tenant predicate is applied in staging and inherited downstream, so this
|
||||
-- test should be impossible to fail. That is precisely why it exists: an
|
||||
-- isolation guarantee nobody tests is the one that turns out to have four
|
||||
-- holes rather than the one reported.
|
||||
select tenant_id, count(*) as rows_from_another_tenant
|
||||
from {{ ref('mart_sales_daily') }}
|
||||
where tenant_id != '{{ var("siax_tenant_id") }}'
|
||||
group by tenant_id
|
||||
Reference in New Issue
Block a user