fix(recovery): restore source removed by production documentation merge
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
/** Shared singleton for the canonical-fact routes (same pattern as source-registry.ts). */
|
||||
import { CanonicalFactService } from "./canonical-fact-service";
|
||||
|
||||
export const canonicalFactService = new CanonicalFactService();
|
||||
@@ -0,0 +1,165 @@
|
||||
/**
|
||||
* src/services/canonical-fact-service.ts
|
||||
* ======================================
|
||||
* 0pt's canonical structured-observation write/read surface (sql/022).
|
||||
*
|
||||
* Ownership boundary: 0pt stores normalized OBSERVED facts (what a source
|
||||
* said, when, from which artifact). L0RE owns epistemic propositions and
|
||||
* projections and must consume these rows through this API. Nothing here
|
||||
* overwrites an earlier observation: the unique key includes source_ref +
|
||||
* observed_at, so conflicting sources are preserved as separate rows.
|
||||
*
|
||||
* Idempotency: identical re-writes hit ON CONFLICT DO NOTHING and are
|
||||
* reported as `duplicates`, never duplicated.
|
||||
*/
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import { SiaxError } from "../platform/errors";
|
||||
import {
|
||||
createCanonicalFactRepository,
|
||||
entityKey,
|
||||
type CanonicalFactRepository,
|
||||
} from "../repositories/canonical-fact-repository";
|
||||
import type {
|
||||
CanonicalEntity,
|
||||
CanonicalEntityInput,
|
||||
CanonicalFact,
|
||||
CanonicalFactInput,
|
||||
CanonicalQuery,
|
||||
CanonicalRelationInput,
|
||||
CanonicalWriteRequest,
|
||||
CanonicalWriteResult,
|
||||
FactValueType,
|
||||
} from "../types/canonical-facts";
|
||||
|
||||
const FACT_VALUE_TYPES: FactValueType[] = ["string", "number", "boolean", "date", "json"];
|
||||
|
||||
function inferValueType(value: unknown): FactValueType {
|
||||
if (typeof value === "string") return "string";
|
||||
if (typeof value === "number") return "number";
|
||||
if (typeof value === "boolean") return "boolean";
|
||||
return "json";
|
||||
}
|
||||
|
||||
function assertEntity(entity: CanonicalEntityInput, label: string): void {
|
||||
if (!entity || !entity.entity_type || !entity.identity_namespace || !entity.identity_value) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", `${label} requires entity_type, identity_namespace and identity_value`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertFact(fact: CanonicalFactInput, index: number): void {
|
||||
assertEntity(fact.entity, `facts[${index}].entity`);
|
||||
if (!fact.predicate) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", `facts[${index}].predicate is required`);
|
||||
if (!fact.source_ref) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", `facts[${index}].source_ref is required`);
|
||||
if (!fact.extractor_version) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", `facts[${index}].extractor_version is required`);
|
||||
if (!fact.observed_at || Number.isNaN(Date.parse(fact.observed_at))) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", `facts[${index}].observed_at must be an ISO timestamp`);
|
||||
}
|
||||
if (fact.value === undefined) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", `facts[${index}].value is required`);
|
||||
if (fact.value_type && !FACT_VALUE_TYPES.includes(fact.value_type)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", `facts[${index}].value_type is invalid`);
|
||||
}
|
||||
if (fact.confidence !== undefined && (fact.confidence < 0 || fact.confidence > 1)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", `facts[${index}].confidence must be between 0 and 1`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertRelation(relation: CanonicalRelationInput, index: number): void {
|
||||
assertEntity(relation.subject, `relations[${index}].subject`);
|
||||
assertEntity(relation.object, `relations[${index}].object`);
|
||||
if (!relation.predicate) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_relation", `relations[${index}].predicate is required`);
|
||||
if (!relation.source_ref) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_relation", `relations[${index}].source_ref is required`);
|
||||
if (!relation.extractor_version) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_relation", `relations[${index}].extractor_version is required`);
|
||||
if (!relation.observed_at || Number.isNaN(Date.parse(relation.observed_at))) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.canonical_relation", `relations[${index}].observed_at must be an ISO timestamp`);
|
||||
}
|
||||
}
|
||||
|
||||
export interface CanonicalFactServiceOptions {
|
||||
repository?: CanonicalFactRepository;
|
||||
aud0BaseUrl?: string;
|
||||
databaseUrl?: string;
|
||||
}
|
||||
|
||||
export class CanonicalFactService {
|
||||
private readonly repo: CanonicalFactRepository;
|
||||
private readonly aud0: Aud0Client;
|
||||
|
||||
constructor(opts?: CanonicalFactServiceOptions) {
|
||||
this.repo = opts?.repository ?? createCanonicalFactRepository(opts?.databaseUrl);
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
}
|
||||
|
||||
async write(input: CanonicalWriteRequest): Promise<CanonicalWriteResult> {
|
||||
if (!input.tenant_id) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", "tenant_id is required");
|
||||
const facts = input.facts ?? [];
|
||||
const relations = input.relations ?? [];
|
||||
const explicit = input.entities ?? [];
|
||||
if (facts.length === 0 && relations.length === 0 && explicit.length === 0) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", "at least one entity, fact or relation is required");
|
||||
}
|
||||
facts.forEach(assertFact);
|
||||
relations.forEach(assertRelation);
|
||||
|
||||
// Unique entity inputs from explicit + facts + relations.
|
||||
const unique = new Map<string, CanonicalEntityInput>();
|
||||
for (const e of [...explicit, ...facts.map((f) => f.entity), ...relations.flatMap((r) => [r.subject, r.object])]) {
|
||||
assertEntity(e, "entity");
|
||||
unique.set(entityKey(e), e);
|
||||
}
|
||||
const entities: CanonicalEntity[] = await this.repo.upsertEntities(input.tenant_id, [...unique.values()]);
|
||||
const entityIds = new Map(entities.map((e) => [entityKey(e), e.entity_id]));
|
||||
|
||||
const normalizedFacts = facts.map((f) => ({ ...f, value_type: f.value_type ?? inferValueType(f.value) }));
|
||||
const factResult = await this.repo.insertFacts(input.tenant_id, normalizedFacts, entityIds);
|
||||
const relationResult = await this.repo.insertRelations(input.tenant_id, relations, entityIds);
|
||||
|
||||
// AUD0: counts and provenance refs only, never fact values (PII discipline).
|
||||
try {
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_canonical_facts_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
|
||||
event_type: "0pt.canonical_facts_written",
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id ?? input.tenant_id,
|
||||
actor_id: "0pt",
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.canonical_fact_write",
|
||||
resource_type: "canonical_facts",
|
||||
resource_id: input.tenant_id,
|
||||
risk_level: "L2",
|
||||
created_at: new Date().toISOString(),
|
||||
payload: {
|
||||
entities: entities.length,
|
||||
facts_inserted: factResult.inserted,
|
||||
facts_duplicates: factResult.duplicates,
|
||||
relations_inserted: relationResult.inserted,
|
||||
relations_duplicates: relationResult.duplicates,
|
||||
source_refs: [...new Set([...facts.map((f) => f.source_ref), ...relations.map((r) => r.source_ref)])],
|
||||
run_refs: [...new Set([...facts.map((f) => f.run_ref), ...relations.map((r) => r.run_ref)].filter(Boolean))],
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
// AUD0 outage must not lose the canonical write; the caller sees counts.
|
||||
}
|
||||
|
||||
return {
|
||||
entities,
|
||||
facts: normalizedFacts.map((f) => ({
|
||||
fact_id: "",
|
||||
entity_id: entityIds.get(entityKey(f.entity)) ?? "",
|
||||
predicate: f.predicate,
|
||||
})),
|
||||
relations: relations.map((r) => ({
|
||||
relation_id: "",
|
||||
subject_id: entityIds.get(entityKey(r.subject)) ?? "",
|
||||
predicate: r.predicate,
|
||||
object_id: entityIds.get(entityKey(r.object)) ?? "",
|
||||
})),
|
||||
duplicates: factResult.duplicates + relationResult.duplicates,
|
||||
};
|
||||
}
|
||||
|
||||
async query(query: CanonicalQuery): Promise<CanonicalFact[]> {
|
||||
if (!query.tenant_id) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", "tenant_id is required");
|
||||
return this.repo.queryFacts({ ...query, limit: Math.min(query.limit ?? 100, 500) });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
import type {
|
||||
DatasetIcebergMapping,
|
||||
DatasetIcebergMappingStatus,
|
||||
CreateDatasetIcebergMappingInput,
|
||||
} from "../types/dataset-iceberg-mapping";
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import { DatasetService } from "./dataset-service";
|
||||
|
||||
interface DatasetIcebergMappingRecord {
|
||||
dataset_iceberg_mapping_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
dataset_id: string;
|
||||
iceberg_table_id: string;
|
||||
namespace_id: string | null;
|
||||
namespace: string | null;
|
||||
table_name: string | null;
|
||||
mapping_status: string;
|
||||
mapped_at: string;
|
||||
mapped_by: string | null;
|
||||
correlation_id: string | null;
|
||||
created_at: string;
|
||||
updated_at: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns 0pt.dataset_iceberg_mapping.schema.v1 -- the pointer from a 0pt
|
||||
* dataset to the Iceberg table it writes to. `iceberg_table_id` is always a
|
||||
* REFERENCE into ST0RE's table registry (never a copy of ST0RE's table row);
|
||||
* this service holds no table/catalog/snapshot state itself. See
|
||||
* docs/ICEBERG_WRITE_BOUNDARY.md.
|
||||
*
|
||||
* This service does not call ST0RE directly -- it records a mapping to an
|
||||
* `iceberg_table_id` that already exists in ST0RE's registry.
|
||||
* `IcebergWriteService.mapDatasetToIcebergTable` (src/services/
|
||||
* iceberg-write-service.ts) is the higher-level flow that registers a NEW
|
||||
* table in ST0RE first and then calls this service to record the mapping --
|
||||
* kept as two composable steps so a mapping can also be created against an
|
||||
* already-registered table (e.g. one N0D or another 0pt job already set up).
|
||||
*/
|
||||
export class DatasetIcebergMappingService {
|
||||
private mappings: Map<string, DatasetIcebergMappingRecord> = new Map();
|
||||
private aud0: Aud0Client;
|
||||
private datasetService: DatasetService;
|
||||
|
||||
constructor(opts?: { aud0BaseUrl?: string; datasetService?: DatasetService }) {
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
this.datasetService = opts?.datasetService ?? new DatasetService(opts?.aud0BaseUrl);
|
||||
}
|
||||
|
||||
async createMapping(input: CreateDatasetIcebergMappingInput): Promise<DatasetIcebergMapping> {
|
||||
if (this.mappings.has(input.dataset_iceberg_mapping_id)) {
|
||||
throw new Error(`Dataset Iceberg mapping ${input.dataset_iceberg_mapping_id} already exists`);
|
||||
}
|
||||
if (!input.iceberg_table_id) {
|
||||
throw new Error("iceberg_table_id is required (reference into ST0RE's table registry)");
|
||||
}
|
||||
// Verify the dataset exists -- a mapping with no dataset is meaningless.
|
||||
try {
|
||||
await this.datasetService.getDataset(input.dataset_id);
|
||||
} catch {
|
||||
throw new Error(`Dataset ${input.dataset_id} not found`);
|
||||
}
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const record: DatasetIcebergMappingRecord = {
|
||||
dataset_iceberg_mapping_id: input.dataset_iceberg_mapping_id,
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
dataset_id: input.dataset_id,
|
||||
iceberg_table_id: input.iceberg_table_id,
|
||||
namespace_id: input.namespace_id ?? null,
|
||||
namespace: input.namespace ?? null,
|
||||
table_name: input.table_name ?? null,
|
||||
mapping_status: input.mapping_status ?? "active",
|
||||
mapped_at: now,
|
||||
mapped_by: input.mapped_by ?? null,
|
||||
correlation_id: input.correlation_id ?? null,
|
||||
created_at: now,
|
||||
updated_at: null,
|
||||
};
|
||||
this.mappings.set(input.dataset_iceberg_mapping_id, record);
|
||||
|
||||
// 0pt.dataset_iceberg_table_mapped -- docs/11-lakehouse-write-plan.md event list,
|
||||
// docs/ICEBERG_WRITE_BOUNDARY.md. Fired when the mapping is created/updated.
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_dataset_iceberg_table_mapped_${input.dataset_iceberg_mapping_id}`,
|
||||
event_type: "0pt.dataset_iceberg_table_mapped",
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
actor_id: input.mapped_by ?? "system",
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.map_dataset_iceberg_table",
|
||||
resource_type: "dataset_iceberg_mapping",
|
||||
resource_id: input.dataset_iceberg_mapping_id,
|
||||
risk_level: "L2",
|
||||
payload: {
|
||||
dataset_id: input.dataset_id,
|
||||
iceberg_table_id: input.iceberg_table_id,
|
||||
namespace_id: input.namespace_id,
|
||||
namespace: input.namespace,
|
||||
table_name: input.table_name,
|
||||
},
|
||||
created_at: now,
|
||||
correlation_id: input.correlation_id ?? "",
|
||||
});
|
||||
|
||||
// Lineage: dataset -> iceberg table.
|
||||
await this.aud0.ingestLineageEvent({
|
||||
lineage_event_id: `lineage_0pt_dataset_iceberg_${input.dataset_iceberg_mapping_id}`,
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.map_dataset_iceberg_table",
|
||||
source_type: "dataset",
|
||||
source_id: input.dataset_id,
|
||||
target_type: "iceberg_table",
|
||||
target_id: input.iceberg_table_id,
|
||||
lineage_type: "stored_as",
|
||||
metadata: { dataset_iceberg_mapping_id: input.dataset_iceberg_mapping_id },
|
||||
created_at: now,
|
||||
correlation_id: input.correlation_id ?? "",
|
||||
});
|
||||
|
||||
return this.toMapping(record);
|
||||
}
|
||||
|
||||
// P014: `tenantId` is optional so trusted in-process callers (existing
|
||||
// unit tests, and internal callers that already resolved the tenant some
|
||||
// other way) are unaffected -- but both HTTP routes that reach this method
|
||||
// now always pass it, closing a real gap: `getMapping` used to be
|
||||
// reachable by mapping ID alone with NO tenant check anywhere, so any
|
||||
// authenticated caller could read -- or, via requestIcebergWrite below,
|
||||
// WRITE a new snapshot onto -- another tenant's dataset<->Iceberg-table
|
||||
// mapping just by guessing/enumerating dataset_iceberg_mapping_id. A
|
||||
// mismatched tenantId reports identically to "not found" (no existence
|
||||
// oracle), matching SourceRegistryService.assertSourceTenant's convention.
|
||||
async getMapping(mappingId: string, tenantId?: string): Promise<DatasetIcebergMapping> {
|
||||
const record = this.mappings.get(mappingId);
|
||||
if (!record || (tenantId && record.tenant_id !== tenantId)) {
|
||||
throw new Error(`Dataset Iceberg mapping ${mappingId} not found`);
|
||||
}
|
||||
return this.toMapping(record);
|
||||
}
|
||||
|
||||
async getMappingForDataset(datasetId: string): Promise<DatasetIcebergMapping | undefined> {
|
||||
const record = Array.from(this.mappings.values()).find((r) => r.dataset_id === datasetId && r.mapping_status === "active");
|
||||
return record ? this.toMapping(record) : undefined;
|
||||
}
|
||||
|
||||
async listMappings(filter?: {
|
||||
tenant_id?: string;
|
||||
project_id?: string;
|
||||
dataset_id?: string;
|
||||
mapping_status?: DatasetIcebergMappingStatus;
|
||||
}): Promise<DatasetIcebergMapping[]> {
|
||||
let results = Array.from(this.mappings.values());
|
||||
if (filter?.tenant_id) results = results.filter((r) => r.tenant_id === filter.tenant_id);
|
||||
if (filter?.project_id) results = results.filter((r) => r.project_id === filter.project_id);
|
||||
if (filter?.dataset_id) results = results.filter((r) => r.dataset_id === filter.dataset_id);
|
||||
if (filter?.mapping_status) results = results.filter((r) => r.mapping_status === filter.mapping_status);
|
||||
return results.map((r) => this.toMapping(r));
|
||||
}
|
||||
|
||||
private toMapping(r: DatasetIcebergMappingRecord): DatasetIcebergMapping {
|
||||
return {
|
||||
dataset_iceberg_mapping_id: r.dataset_iceberg_mapping_id,
|
||||
tenant_id: r.tenant_id,
|
||||
project_id: r.project_id,
|
||||
dataset_id: r.dataset_id,
|
||||
iceberg_table_id: r.iceberg_table_id,
|
||||
namespace_id: r.namespace_id ?? undefined,
|
||||
namespace: r.namespace ?? undefined,
|
||||
table_name: r.table_name ?? undefined,
|
||||
mapping_status: r.mapping_status as DatasetIcebergMappingStatus,
|
||||
mapped_at: r.mapped_at,
|
||||
mapped_by: r.mapped_by ?? undefined,
|
||||
correlation_id: r.correlation_id ?? undefined,
|
||||
created_at: r.created_at,
|
||||
updated_at: r.updated_at ?? undefined,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
/**
|
||||
* Owns 0pt.dataset.schema.v1 — the canonical registry of 0pt datasets.
|
||||
*
|
||||
* Storage is Postgres-backed through DatasetRepository whenever DATABASE_URL
|
||||
* is configured. Non-production tests/local development may use the in-memory
|
||||
* repository; production without a persistent store fails at construction via
|
||||
* resolveStoreMode. Dataset content remains outside this registry.
|
||||
*/
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import {
|
||||
createDatasetRepository,
|
||||
type DatasetListFilter,
|
||||
type DatasetRepository,
|
||||
} from "../repositories/dataset-repository";
|
||||
import type { Dataset, DatasetStatus, DatasetType, CreateDatasetInput } from "../types/dataset";
|
||||
|
||||
export type { Dataset, DatasetStatus, DatasetType, CreateDatasetInput };
|
||||
export type RegisterDatasetInput = CreateDatasetInput;
|
||||
|
||||
export interface DatasetServiceOptions {
|
||||
aud0BaseUrl?: string;
|
||||
databaseUrl?: string;
|
||||
repository?: DatasetRepository;
|
||||
}
|
||||
|
||||
function normalizeOptions(input?: string | DatasetServiceOptions): DatasetServiceOptions {
|
||||
return typeof input === "string" ? { aud0BaseUrl: input } : input ?? {};
|
||||
}
|
||||
|
||||
export class DatasetService {
|
||||
private readonly repo: DatasetRepository;
|
||||
private readonly aud0: Aud0Client;
|
||||
|
||||
/**
|
||||
* A string argument remains supported as the historical aud0BaseUrl form;
|
||||
* new callers should use the options object so repository/database injection
|
||||
* is explicit and testable.
|
||||
*/
|
||||
constructor(input?: string | DatasetServiceOptions) {
|
||||
const opts = normalizeOptions(input);
|
||||
this.aud0 = createAud0Client(opts.aud0BaseUrl);
|
||||
this.repo = opts.repository ?? createDatasetRepository(opts.databaseUrl);
|
||||
}
|
||||
|
||||
async registerDataset(input: CreateDatasetInput): Promise<Dataset> {
|
||||
const dataset = await this.repo.create(input);
|
||||
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_dataset_registered_${dataset.dataset_id}`,
|
||||
event_type: "0pt.dataset_registered",
|
||||
tenant_id: dataset.tenant_id,
|
||||
project_id: dataset.project_id,
|
||||
actor_id: dataset.created_by ?? "system",
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.register_dataset",
|
||||
resource_type: "dataset",
|
||||
resource_id: dataset.dataset_id,
|
||||
risk_level: "L2",
|
||||
payload: { name: dataset.name, dataset_type: dataset.dataset_type },
|
||||
created_at: dataset.created_at,
|
||||
correlation_id: dataset.correlation_id,
|
||||
idempotency_key: `dataset-register:${dataset.tenant_id}:${dataset.dataset_id}`,
|
||||
});
|
||||
|
||||
return dataset;
|
||||
}
|
||||
|
||||
async getDataset(datasetId: string): Promise<Dataset> {
|
||||
return this.repo.get(datasetId);
|
||||
}
|
||||
|
||||
async listDatasets(filter?: {
|
||||
tenant_id?: string;
|
||||
project_id?: string;
|
||||
dataset_type?: DatasetType;
|
||||
status?: DatasetStatus;
|
||||
}): Promise<Dataset[]> {
|
||||
return this.repo.list(filter as DatasetListFilter | undefined);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
/**
|
||||
* Dataset version/query/export capability.
|
||||
*
|
||||
* Dataset metadata is canonical in DatasetService; immutable version metadata
|
||||
* is persisted through DatasetVersionRepository; bytes remain canonical ST0RE
|
||||
* artifacts. AUD0/OpenLineage record the evidence chain.
|
||||
*/
|
||||
import crypto from "crypto";
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import { createSt0reClient, type St0reClient } from "../clients/st0re-client";
|
||||
import { LineageEmitter } from "../lineage/lineage-emitter";
|
||||
import {
|
||||
createDatasetVersionRepository,
|
||||
type DatasetVersionRepository,
|
||||
} from "../repositories/dataset-version-repository";
|
||||
import { SiaxError } from "../platform/errors";
|
||||
import { formatResourceRef, isResourceRef } from "../platform/resource-ref";
|
||||
import type {
|
||||
CreateDatasetVersionInput,
|
||||
DatasetExport,
|
||||
DatasetVersion,
|
||||
} from "../types/dataset-version";
|
||||
import { DatasetService } from "./dataset-service";
|
||||
|
||||
export type { CreateDatasetVersionInput, DatasetExport, DatasetVersion } from "../types/dataset-version";
|
||||
|
||||
export interface DatasetVersionServiceOptions {
|
||||
aud0BaseUrl?: string;
|
||||
st0reBaseUrl?: string;
|
||||
databaseUrl?: string;
|
||||
datasetService?: DatasetService;
|
||||
repository?: DatasetVersionRepository;
|
||||
}
|
||||
|
||||
export class DatasetVersionService {
|
||||
private readonly repo: DatasetVersionRepository;
|
||||
private readonly aud0: Aud0Client;
|
||||
private readonly st0re: St0reClient;
|
||||
private readonly lineage: LineageEmitter;
|
||||
readonly datasets: DatasetService;
|
||||
|
||||
constructor(opts?: DatasetVersionServiceOptions) {
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
this.st0re = createSt0reClient(opts?.st0reBaseUrl);
|
||||
this.lineage = new LineageEmitter(this.aud0);
|
||||
this.datasets = opts?.datasetService ?? new DatasetService({
|
||||
aud0BaseUrl: opts?.aud0BaseUrl,
|
||||
databaseUrl: opts?.databaseUrl,
|
||||
});
|
||||
this.repo = opts?.repository ?? createDatasetVersionRepository(opts?.databaseUrl);
|
||||
}
|
||||
|
||||
async createVersion(input: CreateDatasetVersionInput): Promise<DatasetVersion> {
|
||||
await this.assertDatasetTenant(input.dataset_id, input.tenant_id);
|
||||
if (!input.project_id?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "project_id is required");
|
||||
if (!input.schema_version?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "schema_version is required");
|
||||
if (!input.created_by?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "created_by is required");
|
||||
if (input.record_count !== undefined && (!Number.isSafeInteger(input.record_count) || input.record_count < 0)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "record_count must be a non-negative safe integer");
|
||||
}
|
||||
if (input.artifact_refs.length === 0) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "a dataset version needs at least one artifact ref");
|
||||
}
|
||||
for (const ref of input.artifact_refs) {
|
||||
if (!isResourceRef(ref, "st0re", "artifact")) {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.dataset",
|
||||
`artifact ref "${ref}" is not a canonical st0re:artifact:... reference (Master-DoD §5)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const version = await this.repo.create(input);
|
||||
const versionRef = formatResourceRef({
|
||||
product: "0pt",
|
||||
resourceType: "dataset",
|
||||
resourceId: input.dataset_id,
|
||||
version: String(version.version),
|
||||
});
|
||||
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_dataset_version_${input.dataset_id}_v${version.version}`,
|
||||
event_type: "0pt.dataset_version_created",
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
actor_id: input.created_by,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.dataset.version.v1",
|
||||
resource_type: "dataset_version",
|
||||
resource_id: `${input.dataset_id}@${version.version}`,
|
||||
risk_level: "L2",
|
||||
result: "success",
|
||||
input_refs: version.artifact_refs,
|
||||
output_refs: [versionRef],
|
||||
payload: { schema_version: version.schema_version, record_count: version.record_count },
|
||||
created_at: version.created_at,
|
||||
occurred_at: version.created_at,
|
||||
correlation_id: version.correlation_id,
|
||||
idempotency_key: `dataset-version:${input.tenant_id}:${input.dataset_id}:v${version.version}`,
|
||||
});
|
||||
await this.lineage.emit({
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
capability_id: "0pt.dataset.version.v1",
|
||||
run_id: `dataset-version:${input.dataset_id}:v${version.version}`,
|
||||
event_type: "COMPLETE",
|
||||
event_time: version.created_at,
|
||||
inputs: version.artifact_refs.map((ref) => ({ name: ref, ref })),
|
||||
outputs: [{ name: `dataset/${input.dataset_id}`, ref: versionRef }],
|
||||
correlation_id: version.correlation_id,
|
||||
});
|
||||
|
||||
return version;
|
||||
}
|
||||
|
||||
async listVersions(datasetId: string, tenantId: string): Promise<DatasetVersion[]> {
|
||||
await this.assertDatasetTenant(datasetId, tenantId);
|
||||
return this.repo.list(datasetId, tenantId);
|
||||
}
|
||||
|
||||
async getVersion(datasetId: string, tenantId: string, version: number): Promise<DatasetVersion> {
|
||||
await this.assertDatasetTenant(datasetId, tenantId);
|
||||
if (!Number.isSafeInteger(version) || version < 1) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "version must be a positive integer");
|
||||
}
|
||||
return this.repo.get(datasetId, tenantId, version);
|
||||
}
|
||||
|
||||
async exportDataset(input: {
|
||||
dataset_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
version: number;
|
||||
format: DatasetExport["format"];
|
||||
object_location: { provider: string; bucket: string; key: string; region?: string };
|
||||
size_bytes: number;
|
||||
checksum_sha256: string;
|
||||
actor_id: string;
|
||||
correlation_id?: string;
|
||||
}): Promise<DatasetExport> {
|
||||
const version = await this.getVersion(input.dataset_id, input.tenant_id, input.version);
|
||||
if (!(["parquet", "csv", "json"] as const).includes(input.format)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "unsupported export format");
|
||||
}
|
||||
if (!Number.isSafeInteger(input.size_bytes) || input.size_bytes < 0) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "size_bytes must be a non-negative safe integer");
|
||||
}
|
||||
if (!/^[a-fA-F0-9]{64}$/.test(input.checksum_sha256)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "checksum_sha256 must be a 64-character hex digest");
|
||||
}
|
||||
|
||||
const exportKey = `dsexp:${input.tenant_id}:${input.dataset_id}:v${input.version}:${input.format}:${input.checksum_sha256.toLowerCase()}`;
|
||||
const exportId = `dsexp_${crypto.createHash("sha256").update(exportKey).digest("hex").slice(0, 32)}`;
|
||||
|
||||
const artifact = await this.st0re.createArtifact({
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
artifact_type: "dataset_export",
|
||||
app_id: "0pt",
|
||||
produced_by: "0pt.dataset.export.v1",
|
||||
object_location: input.object_location,
|
||||
content_type:
|
||||
input.format === "parquet"
|
||||
? "application/vnd.apache.parquet"
|
||||
: input.format === "csv"
|
||||
? "text/csv"
|
||||
: "application/json",
|
||||
size_bytes: input.size_bytes,
|
||||
checksum_sha256: input.checksum_sha256.toLowerCase(),
|
||||
pii_level: "none",
|
||||
retention_policy: "standard",
|
||||
data_residency: "eu",
|
||||
risk_level: "L2",
|
||||
correlation_id: input.correlation_id,
|
||||
created_by: input.actor_id,
|
||||
idempotency_key: exportKey,
|
||||
});
|
||||
|
||||
const artifactRef = formatResourceRef({
|
||||
product: "st0re",
|
||||
resourceType: "artifact",
|
||||
resourceId: artifact.artifact_id,
|
||||
});
|
||||
const createdAt = artifact.created_at ?? new Date().toISOString();
|
||||
const record: DatasetExport = {
|
||||
export_id: exportId,
|
||||
dataset_id: input.dataset_id,
|
||||
version: input.version,
|
||||
tenant_id: input.tenant_id,
|
||||
artifact_ref: artifactRef,
|
||||
format: input.format,
|
||||
created_at: createdAt,
|
||||
};
|
||||
|
||||
const versionRef = formatResourceRef({
|
||||
product: "0pt",
|
||||
resourceType: "dataset",
|
||||
resourceId: input.dataset_id,
|
||||
version: String(version.version),
|
||||
});
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_dataset_export_${exportId}`,
|
||||
event_type: "0pt.dataset_exported",
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
actor_id: input.actor_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.dataset.export.v1",
|
||||
resource_type: "dataset_export",
|
||||
resource_id: exportId,
|
||||
risk_level: "L3",
|
||||
result: "success",
|
||||
input_refs: [versionRef],
|
||||
output_refs: [artifactRef],
|
||||
payload: { format: input.format, size_bytes: input.size_bytes },
|
||||
created_at: createdAt,
|
||||
occurred_at: createdAt,
|
||||
correlation_id: input.correlation_id,
|
||||
idempotency_key: exportKey,
|
||||
});
|
||||
await this.lineage.emit({
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
capability_id: "0pt.dataset.export.v1",
|
||||
run_id: `dataset-export:${exportId}`,
|
||||
event_type: "COMPLETE",
|
||||
event_time: createdAt,
|
||||
inputs: [{ name: `dataset/${input.dataset_id}`, ref: versionRef }],
|
||||
outputs: [{ name: artifactRef, ref: artifactRef }],
|
||||
correlation_id: input.correlation_id,
|
||||
});
|
||||
|
||||
return record;
|
||||
}
|
||||
|
||||
private async assertDatasetTenant(datasetId: string, tenantId: string): Promise<void> {
|
||||
if (!tenantId?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "tenant_id is required");
|
||||
let dataset;
|
||||
try {
|
||||
dataset = await this.datasets.getDataset(datasetId);
|
||||
} catch {
|
||||
throw new SiaxError("NOT_FOUND", "0pt.dataset", `dataset ${datasetId} not found`);
|
||||
}
|
||||
if (dataset.tenant_id !== tenantId) {
|
||||
throw new SiaxError("NOT_FOUND", "0pt.dataset", `dataset ${datasetId} not found`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
/** Shared composition root for the dataset HTTP surface. */
|
||||
import { DatasetService } from "./dataset-service";
|
||||
import { DatasetVersionService } from "./dataset-version-service";
|
||||
|
||||
export const datasetService = new DatasetService();
|
||||
export const datasetVersionService = new DatasetVersionService({ datasetService });
|
||||
@@ -0,0 +1,273 @@
|
||||
/**
|
||||
* src/services/document-capture-service.ts
|
||||
* =========================================
|
||||
* Orchestrates the capture half of the 0pt.document_ocr capability:
|
||||
* poll a customer-registered mailbox (Microsoft Graph, app-only) →
|
||||
* download PDF attachments → persist the bytes as an artifact →
|
||||
* enqueue extraction (an N0D job placeholder; a worker or the
|
||||
* /extraction API route performs the real Document Intelligence call).
|
||||
*
|
||||
* Tenant-agnostic: mailbox address + Graph credentials + folder are all
|
||||
* caller-supplied per invocation — nothing here is stored keyed only by a
|
||||
* hardcoded mailbox. Nothing here assumes invoices, vendors, or Protosell;
|
||||
* that logic (vendor-kb matching, dup-hash, DQ scoring, invoice
|
||||
* classification) was intentionally left behind in
|
||||
* v0-datahub/lib/invoice-inbox — it does not belong in a generic capture
|
||||
* pipeline that also has to serve HR forms, contracts, receipts, etc.
|
||||
*
|
||||
* KNOWN GAP — ST0RE bytes storage: st0re-client.ts registers artifact
|
||||
* METADATA only (a pointer to provider/bucket/key); it has no real
|
||||
* bytes-upload endpoint yet. Until one exists, this service writes
|
||||
* attachment bytes to LocalBlobStore (src/adapters/local-blob-store.ts), a
|
||||
* clearly-marked interim local-disk store, then registers that location
|
||||
* with ST0RE so the artifact/lineage metadata is still correct. Search this
|
||||
* file for "INTERIM" to find the seam to replace.
|
||||
*
|
||||
* Singleton note: unlike source-service.ts / dataset-service.ts (which are
|
||||
* instantiated fresh per API route file because each request is
|
||||
* self-contained), DocumentCaptureService holds the only in-memory record
|
||||
* of *which bytes belong to which document_id* — document-extraction-service
|
||||
* must read the exact same instance to find them. So this module also
|
||||
* exports a shared `documentCaptureService` singleton for both API routes
|
||||
* to import, instead of each route constructing its own instance.
|
||||
*/
|
||||
import crypto from "crypto";
|
||||
import { GraphMailboxAdapter, type GraphMailboxConfig, type GraphMessage, type GraphAttachment } from "../adapters/graph-mailbox-adapter";
|
||||
import { LocalBlobStore } from "../adapters/local-blob-store";
|
||||
import { createSt0reClient, type St0reClient } from "../clients/st0re-client";
|
||||
import { createN0dClient, type N0dClient } from "../clients/n0d-client";
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import type { DocumentRecord, DocumentStatus } from "../types/document";
|
||||
|
||||
export interface CaptureMailboxContext {
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
actor_id: string;
|
||||
correlation_id: string;
|
||||
}
|
||||
|
||||
export interface CaptureMailboxInput {
|
||||
mailbox: GraphMailboxConfig;
|
||||
since_iso?: string;
|
||||
top?: number;
|
||||
max_pages?: number;
|
||||
}
|
||||
|
||||
export interface CaptureMailboxResult {
|
||||
scanned_messages: number;
|
||||
captured_documents: DocumentRecord[];
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
interface StoredBytesLocation {
|
||||
bucket: string;
|
||||
key: string;
|
||||
}
|
||||
|
||||
export class DocumentCaptureService {
|
||||
private documents: Map<string, DocumentRecord> = new Map();
|
||||
private bytesByDocumentId: Map<string, StoredBytesLocation> = new Map();
|
||||
private st0re: St0reClient;
|
||||
private n0d: N0dClient;
|
||||
private aud0: Aud0Client;
|
||||
private blobStore: LocalBlobStore;
|
||||
|
||||
constructor(opts?: { st0reBaseUrl?: string; n0dBaseUrl?: string; aud0BaseUrl?: string; blobStore?: LocalBlobStore }) {
|
||||
this.st0re = createSt0reClient(opts?.st0reBaseUrl);
|
||||
this.n0d = createN0dClient(opts?.n0dBaseUrl);
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
this.blobStore = opts?.blobStore ?? new LocalBlobStore();
|
||||
}
|
||||
|
||||
async captureFromMailbox(input: CaptureMailboxInput, ctx: CaptureMailboxContext): Promise<CaptureMailboxResult> {
|
||||
if (!ctx.tenant_id || !ctx.project_id || !ctx.actor_id) {
|
||||
throw new Error("captureFromMailbox: missing tenant_id/project_id/actor_id");
|
||||
}
|
||||
if (!GraphMailboxAdapter.isConfigured(input.mailbox)) {
|
||||
throw new Error(
|
||||
"captureFromMailbox: missing mailbox.tenantId/clientId/clientSecret/mailboxAddress",
|
||||
);
|
||||
}
|
||||
const graph = new GraphMailboxAdapter(input.mailbox);
|
||||
const result: CaptureMailboxResult = { scanned_messages: 0, captured_documents: [], errors: [] };
|
||||
|
||||
const messages = await graph.pollMessages({
|
||||
sinceISO: input.since_iso,
|
||||
top: input.top,
|
||||
maxPages: input.max_pages,
|
||||
});
|
||||
|
||||
for (const message of messages) {
|
||||
result.scanned_messages++;
|
||||
if (!message.hasAttachments) continue;
|
||||
try {
|
||||
const attachments = await graph.listAttachments(message.id);
|
||||
for (const attachment of attachments) {
|
||||
if (!attachment.isPdf) continue;
|
||||
const document = await this.captureAttachment(input.mailbox, message, attachment, ctx);
|
||||
result.captured_documents.push(document);
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
result.errors.push(`${message.id}: ${msg.slice(0, 200)}`);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private async captureAttachment(
|
||||
mailbox: GraphMailboxConfig,
|
||||
message: GraphMessage,
|
||||
attachment: GraphAttachment,
|
||||
ctx: CaptureMailboxContext,
|
||||
): Promise<DocumentRecord> {
|
||||
const documentId = `doc_${crypto.randomUUID()}`;
|
||||
const now = new Date().toISOString();
|
||||
const bucket = "0pt-documents";
|
||||
const safeName = (attachment.name || "attachment.pdf").replace(/[^A-Za-z0-9._-]/g, "_");
|
||||
const key = `${ctx.tenant_id}/${ctx.project_id}/${message.id}/${attachment.id}/${safeName}`;
|
||||
|
||||
// INTERIM — see file header. Real bytes persistence until ST0RE exposes an upload endpoint.
|
||||
const written = await this.blobStore.write(bucket, key, attachment.contentBytes);
|
||||
const checksum = crypto.createHash("sha256").update(attachment.contentBytes).digest("hex");
|
||||
|
||||
let record: DocumentRecord = {
|
||||
document_id: documentId,
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
mailbox_address: mailbox.mailboxAddress,
|
||||
graph_message_id: message.id,
|
||||
graph_attachment_id: attachment.id,
|
||||
filename: attachment.name ?? undefined,
|
||||
content_type: attachment.contentType ?? undefined,
|
||||
size_bytes: attachment.sizeBytes ?? attachment.contentBytes.length,
|
||||
received_at: message.receivedAt ?? undefined,
|
||||
sender_email: message.senderEmail ?? undefined,
|
||||
sender_name: message.senderName ?? undefined,
|
||||
subject: message.subject ?? undefined,
|
||||
status: "captured",
|
||||
correlation_id: ctx.correlation_id,
|
||||
created_at: now,
|
||||
};
|
||||
this.documents.set(documentId, record);
|
||||
this.bytesByDocumentId.set(documentId, { bucket: written.bucket, key: written.key });
|
||||
|
||||
// Register the artifact pointer with ST0RE (metadata only — see file header gap note).
|
||||
const artifact = await this.st0re.createArtifact({
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
artifact_type: "document",
|
||||
app_id: "0pt",
|
||||
produced_by: "0pt.capture_document",
|
||||
object_location: { provider: written.provider, bucket: written.bucket, key: written.key, region: "local" },
|
||||
content_type: attachment.contentType || "application/pdf",
|
||||
size_bytes: written.sizeBytes,
|
||||
checksum_sha256: checksum,
|
||||
pii_level: "high",
|
||||
retention_policy: "document_capture_standard",
|
||||
data_residency: "eu",
|
||||
risk_level: "L3",
|
||||
correlation_id: ctx.correlation_id,
|
||||
});
|
||||
|
||||
// Enqueue extraction. A worker (out of scope here) or a caller hitting the
|
||||
// /documents/[documentId]/extraction route performs the real DI call.
|
||||
const job = await this.n0d.createJob({
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
job_type: "0pt.document_extraction_job",
|
||||
capability_id: "0pt.extract_document",
|
||||
requested_by_app_id: "0pt",
|
||||
requested_by_actor_id: ctx.actor_id,
|
||||
input_artifact_ids: [artifact.artifact_id],
|
||||
output_artifact_type: "document_extraction",
|
||||
requires_backbone: true,
|
||||
requires_gpu: false,
|
||||
required_node_roles: ["compute_general"],
|
||||
resource_profile: "cpu_io_medium",
|
||||
risk_level: "L3",
|
||||
correlation_id: ctx.correlation_id,
|
||||
});
|
||||
|
||||
record = {
|
||||
...record,
|
||||
st0re_artifact_id: artifact.artifact_id,
|
||||
n0d_job_id: job.job_id,
|
||||
status: "extraction_requested",
|
||||
updated_at: new Date().toISOString(),
|
||||
};
|
||||
this.documents.set(documentId, record);
|
||||
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_document_captured_${documentId}`,
|
||||
event_type: "0pt.document_captured",
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
actor_id: ctx.actor_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.capture_document",
|
||||
resource_type: "document",
|
||||
resource_id: documentId,
|
||||
risk_level: "L3",
|
||||
payload: {
|
||||
mailbox_address: mailbox.mailboxAddress,
|
||||
graph_message_id: message.id,
|
||||
filename: attachment.name,
|
||||
size_bytes: written.sizeBytes,
|
||||
st0re_artifact_id: artifact.artifact_id,
|
||||
n0d_job_id: job.job_id,
|
||||
},
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: ctx.correlation_id,
|
||||
});
|
||||
|
||||
await this.aud0.ingestLineageEvent({
|
||||
lineage_event_id: `lineage_0pt_document_captured_${documentId}`,
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.capture_document",
|
||||
source_type: "mailbox",
|
||||
source_id: mailbox.mailboxAddress,
|
||||
target_type: "document",
|
||||
target_id: documentId,
|
||||
lineage_type: "produced",
|
||||
metadata: { graph_message_id: message.id, st0re_artifact_id: artifact.artifact_id },
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: ctx.correlation_id,
|
||||
});
|
||||
|
||||
return record;
|
||||
}
|
||||
|
||||
async getDocument(documentId: string): Promise<DocumentRecord> {
|
||||
const record = this.documents.get(documentId);
|
||||
if (!record) throw new Error(`Document ${documentId} not found`);
|
||||
return record;
|
||||
}
|
||||
|
||||
async listDocuments(filter?: { tenant_id?: string; project_id?: string; status?: DocumentStatus }): Promise<DocumentRecord[]> {
|
||||
let results = Array.from(this.documents.values());
|
||||
if (filter?.tenant_id) results = results.filter((r) => r.tenant_id === filter.tenant_id);
|
||||
if (filter?.project_id) results = results.filter((r) => r.project_id === filter.project_id);
|
||||
if (filter?.status) results = results.filter((r) => r.status === filter.status);
|
||||
return results;
|
||||
}
|
||||
|
||||
/** Read back the raw bytes captured for a document. INTERIM local store — see file header. */
|
||||
async getDocumentBytes(documentId: string): Promise<Buffer> {
|
||||
const loc = this.bytesByDocumentId.get(documentId);
|
||||
if (!loc) throw new Error(`Document ${documentId} has no stored bytes`);
|
||||
return this.blobStore.read(loc.bucket, loc.key);
|
||||
}
|
||||
|
||||
setStatus(documentId: string, status: DocumentStatus): void {
|
||||
const record = this.documents.get(documentId);
|
||||
if (!record) throw new Error(`Document ${documentId} not found`);
|
||||
record.status = status;
|
||||
record.updated_at = new Date().toISOString();
|
||||
}
|
||||
}
|
||||
|
||||
/** Shared singleton — see the "Singleton note" in the file header for why. */
|
||||
export const documentCaptureService = new DocumentCaptureService();
|
||||
@@ -0,0 +1,290 @@
|
||||
/**
|
||||
* src/services/document-extraction-service.ts
|
||||
* ==============================================
|
||||
* Evidence-based field extraction for a captured document.
|
||||
*
|
||||
* Calls DocumentIntelligenceAdapter and NEVER fabricates a value: a field
|
||||
* key is only populated with a non-null value when Document Intelligence
|
||||
* actually returned one, and it always carries DI's own confidence score
|
||||
* (or null when DI reported none). Fields DI did not find are left blank —
|
||||
* { value: null, confidence: <DI's confidence if any>, found: false } —
|
||||
* never defaulted, guessed, or backfilled from anything else. This mirrors
|
||||
* the evidence-based discipline in
|
||||
* v0-datahub/lib/invoice-inbox/doc-intelligence.ts exactly (it never
|
||||
* invented an amount/vendor/date that DI didn't return); it is a
|
||||
* correctness property, not a style preference, so it is enforced twice —
|
||||
* once inside the adapter's field normalizer, and again defensively here.
|
||||
*
|
||||
* modelId is caller-supplied ("prebuilt-invoice", "prebuilt-receipt",
|
||||
* "prebuilt-layout", or a custom model id) — this service does not assume
|
||||
* any particular document type. Invoice-specific reasoning (vendor
|
||||
* matching, dup-hash, DQ scoring, invoice classification) was intentionally
|
||||
* left in v0-datahub/lib/invoice-inbox and is NOT ported here.
|
||||
*/
|
||||
import crypto from "crypto";
|
||||
import {
|
||||
DocumentIntelligenceAdapter,
|
||||
type DocumentIntelligenceConfig,
|
||||
type ExtractedField as AdapterField,
|
||||
} from "../adapters/document-intelligence-adapter";
|
||||
import { LocalBlobStore } from "../adapters/local-blob-store";
|
||||
import { createCl0udClient, type Cl0udClient } from "../clients/cl0ud-client";
|
||||
import { createSt0reClient, type St0reClient } from "../clients/st0re-client";
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import { DocumentCaptureService, documentCaptureService } from "./document-capture-service";
|
||||
import type { DocumentExtractionRecord, ExtractedField } from "../types/document-extraction";
|
||||
|
||||
export interface ExtractDocumentContext {
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
actor_id: string;
|
||||
correlation_id: string;
|
||||
/** e.g. "prebuilt-invoice" | "prebuilt-receipt" | "prebuilt-layout" | a custom model id. */
|
||||
model_id: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Defensive re-application of the "never fabricate" rule at the service
|
||||
* boundary. Even though the adapter's normalizeField() already enforces
|
||||
* this, a field is only ever considered found here if it is BOTH marked
|
||||
* found by the adapter AND carries a genuinely non-empty value — so a
|
||||
* malformed/partial adapter response can never leak a fabricated value
|
||||
* through this service.
|
||||
*/
|
||||
function toEvidenceFields(fields: Record<string, AdapterField>): Record<string, ExtractedField> {
|
||||
const out: Record<string, ExtractedField> = {};
|
||||
for (const [key, f] of Object.entries(fields)) {
|
||||
const found = !!f?.found && f.value !== null && f.value !== undefined && f.value !== "";
|
||||
out[key] = { value: found ? f.value : null, confidence: f?.confidence ?? null, found };
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export class DocumentExtractionService {
|
||||
private extractions: Map<string, DocumentExtractionRecord> = new Map();
|
||||
private cl0ud: Cl0udClient;
|
||||
private st0re: St0reClient;
|
||||
private aud0: Aud0Client;
|
||||
private blobStore: LocalBlobStore;
|
||||
private di: DocumentIntelligenceAdapter | null;
|
||||
|
||||
constructor(
|
||||
private captureService: DocumentCaptureService,
|
||||
opts?: {
|
||||
cl0udBaseUrl?: string;
|
||||
st0reBaseUrl?: string;
|
||||
aud0BaseUrl?: string;
|
||||
documentIntelligence?: DocumentIntelligenceConfig;
|
||||
blobStore?: LocalBlobStore;
|
||||
},
|
||||
) {
|
||||
this.cl0ud = createCl0udClient(opts?.cl0udBaseUrl);
|
||||
this.st0re = createSt0reClient(opts?.st0reBaseUrl);
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
this.blobStore = opts?.blobStore ?? new LocalBlobStore();
|
||||
this.di =
|
||||
opts?.documentIntelligence && DocumentIntelligenceAdapter.isConfigured(opts.documentIntelligence)
|
||||
? new DocumentIntelligenceAdapter(opts.documentIntelligence)
|
||||
: null;
|
||||
}
|
||||
|
||||
async extractDocument(documentId: string, ctx: ExtractDocumentContext): Promise<DocumentExtractionRecord> {
|
||||
if (!this.di) {
|
||||
throw new Error(
|
||||
"DocumentExtractionService: Document Intelligence is not configured (missing endpoint/apiKey) — refusing to fabricate a result",
|
||||
);
|
||||
}
|
||||
if (!ctx.model_id) {
|
||||
throw new Error("extractDocument: missing model_id");
|
||||
}
|
||||
if (!ctx.tenant_id || !ctx.project_id || !ctx.actor_id) {
|
||||
throw new Error("extractDocument: missing tenant_id/project_id/actor_id");
|
||||
}
|
||||
const document = await this.captureService.getDocument(documentId);
|
||||
// The caller's verified tenant alone is insufficient: the document being
|
||||
// read must belong to the same tenant AND project before policy or bytes.
|
||||
if (document.tenant_id !== ctx.tenant_id || document.project_id !== ctx.project_id) {
|
||||
throw new Error(`Document ${documentId} not found`);
|
||||
}
|
||||
|
||||
const policy = await this.cl0ud.evaluatePolicy({
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
actor_id: ctx.actor_id,
|
||||
capability_id: "0pt.extract_document",
|
||||
resource_type: "document",
|
||||
resource_id: documentId,
|
||||
risk_level: "L3",
|
||||
correlation_id: ctx.correlation_id,
|
||||
});
|
||||
if (policy.decision === "deny") throw new Error(`Policy denied: ${policy.reason}`);
|
||||
if (policy.decision === "approval_required") throw new Error(`Policy requires approval: ${policy.reason}`);
|
||||
|
||||
const extractionId = `dex_${crypto.randomUUID()}`;
|
||||
const now = new Date().toISOString();
|
||||
let record: DocumentExtractionRecord = {
|
||||
extraction_id: extractionId,
|
||||
document_id: documentId,
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
model_id: ctx.model_id,
|
||||
status: "requested",
|
||||
fields: {},
|
||||
line_items: [],
|
||||
fields_found_count: 0,
|
||||
fields_total_count: 0,
|
||||
average_confidence: null,
|
||||
document_confidence: null,
|
||||
correlation_id: ctx.correlation_id,
|
||||
created_at: now,
|
||||
};
|
||||
this.extractions.set(extractionId, record);
|
||||
|
||||
try {
|
||||
const bytes = await this.captureService.getDocumentBytes(documentId);
|
||||
const diResult = await this.di.analyzeDocument(bytes, ctx.model_id);
|
||||
|
||||
const fields = toEvidenceFields(diResult.fields);
|
||||
const lineItems = Object.values(diResult.arrayFields).flatMap((items) => items.map((item) => toEvidenceFields(item)));
|
||||
|
||||
const foundCount = Object.values(fields).filter((f) => f.found).length;
|
||||
// Average only over FOUND fields — a not-found field's confidence (e.g. "31% sure this
|
||||
// region isn't a date after all") describes DI's certainty about absence, not a value,
|
||||
// and must not drag down the confidence of the values that were actually extracted.
|
||||
const confidences = Object.values(fields)
|
||||
.filter((f) => f.found)
|
||||
.map((f) => f.confidence)
|
||||
.filter((c): c is number => c != null);
|
||||
const averageConfidence = confidences.length ? confidences.reduce((a, b) => a + b, 0) / confidences.length : null;
|
||||
|
||||
// Persist the extraction result as an artifact too — same interim-storage gap as
|
||||
// document-capture-service.ts (see that file's header for the full explanation).
|
||||
const payloadJson = Buffer.from(
|
||||
JSON.stringify({ model_id: ctx.model_id, fields, line_items: lineItems, content: diResult.content }),
|
||||
"utf8",
|
||||
);
|
||||
const bucket = "0pt-document-extractions";
|
||||
const key = `${ctx.tenant_id}/${ctx.project_id}/${documentId}/${extractionId}.json`;
|
||||
const written = await this.blobStore.write(bucket, key, payloadJson);
|
||||
const artifact = await this.st0re.createArtifact({
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
artifact_type: "document_extraction",
|
||||
app_id: "0pt",
|
||||
produced_by: "0pt.extract_document",
|
||||
object_location: { provider: written.provider, bucket: written.bucket, key: written.key, region: "local" },
|
||||
content_type: "application/json",
|
||||
size_bytes: written.sizeBytes,
|
||||
checksum_sha256: crypto.createHash("sha256").update(payloadJson).digest("hex"),
|
||||
pii_level: "high",
|
||||
retention_policy: "document_extraction_standard",
|
||||
data_residency: "eu",
|
||||
risk_level: "L3",
|
||||
correlation_id: ctx.correlation_id,
|
||||
});
|
||||
|
||||
record = {
|
||||
...record,
|
||||
status: "completed",
|
||||
fields,
|
||||
line_items: lineItems,
|
||||
fields_found_count: foundCount,
|
||||
fields_total_count: Object.keys(fields).length,
|
||||
average_confidence: averageConfidence,
|
||||
document_confidence: diResult.documentConfidence,
|
||||
st0re_artifact_id: artifact.artifact_id,
|
||||
updated_at: new Date().toISOString(),
|
||||
};
|
||||
this.extractions.set(extractionId, record);
|
||||
this.captureService.setStatus(documentId, "extracted");
|
||||
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_document_extracted_${extractionId}`,
|
||||
event_type: "0pt.document_extracted",
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
actor_id: ctx.actor_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.extract_document",
|
||||
resource_type: "document_extraction",
|
||||
resource_id: extractionId,
|
||||
risk_level: "L3",
|
||||
payload: {
|
||||
document_id: documentId,
|
||||
model_id: ctx.model_id,
|
||||
fields_found_count: foundCount,
|
||||
fields_total_count: Object.keys(fields).length,
|
||||
average_confidence: averageConfidence,
|
||||
st0re_artifact_id: artifact.artifact_id,
|
||||
},
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: ctx.correlation_id,
|
||||
});
|
||||
|
||||
await this.aud0.ingestLineageEvent({
|
||||
lineage_event_id: `lineage_0pt_document_extracted_${extractionId}`,
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.extract_document",
|
||||
source_type: "document",
|
||||
source_id: documentId,
|
||||
target_type: "document_extraction",
|
||||
target_id: extractionId,
|
||||
lineage_type: "derived_from",
|
||||
metadata: { model_id: ctx.model_id, st0re_artifact_id: artifact.artifact_id },
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: ctx.correlation_id,
|
||||
});
|
||||
|
||||
return record;
|
||||
} catch (e: unknown) {
|
||||
const message = e instanceof Error ? e.message : String(e);
|
||||
record = { ...record, status: "failed", error: message, updated_at: new Date().toISOString() };
|
||||
this.extractions.set(extractionId, record);
|
||||
try {
|
||||
this.captureService.setStatus(documentId, "extraction_failed");
|
||||
} catch {
|
||||
/* document may already be gone in edge-case tests — extraction failure record still stands */
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async getExtraction(extractionId: string): Promise<DocumentExtractionRecord> {
|
||||
const record = this.extractions.get(extractionId);
|
||||
if (!record) throw new Error(`Document extraction ${extractionId} not found`);
|
||||
return record;
|
||||
}
|
||||
|
||||
// P014: `tenantId` is optional so existing trusted in-process callers are
|
||||
// unaffected, but the HTTP route now always supplies it -- closing a real
|
||||
// gap: this used to be reachable by documentId alone with NO tenant
|
||||
// check anywhere, so any authenticated caller could list another
|
||||
// tenant's OCR/structuring extraction results (potentially containing
|
||||
// extracted document content, e.g. invoice fields) by guessing or
|
||||
// enumerating documentId. A mismatched tenantId reports identically to
|
||||
// "not found" (no existence oracle), same convention as the rest of the
|
||||
// surface.
|
||||
async listExtractionsForDocument(documentId: string, tenantId?: string): Promise<DocumentExtractionRecord[]> {
|
||||
if (tenantId) {
|
||||
const document = await this.captureService.getDocument(documentId).catch(() => undefined);
|
||||
if (!document || document.tenant_id !== tenantId) {
|
||||
throw new Error(`Document ${documentId} not found`);
|
||||
}
|
||||
}
|
||||
return Array.from(this.extractions.values()).filter((r) => r.document_id === documentId);
|
||||
}
|
||||
}
|
||||
|
||||
function buildDocumentIntelligenceConfigFromEnv(): DocumentIntelligenceConfig | undefined {
|
||||
const endpoint = process.env.DOC_INTELLIGENCE_ENDPOINT;
|
||||
const apiKey = process.env.DOC_INTELLIGENCE_KEY;
|
||||
if (!endpoint || !apiKey) return undefined;
|
||||
return { endpoint, apiKey, apiVersion: process.env.DOC_INTELLIGENCE_API_VERSION };
|
||||
}
|
||||
|
||||
/** Shared singleton (see document-capture-service.ts "Singleton note") wired to the shared capture service. */
|
||||
export const documentExtractionService = new DocumentExtractionService(documentCaptureService, {
|
||||
documentIntelligence: buildDocumentIntelligenceConfigFromEnv(),
|
||||
});
|
||||
@@ -0,0 +1,10 @@
|
||||
import { createN0dClient } from "../clients/n0d-client";
|
||||
import { EntityResolutionService } from "../er/entity-resolution-service";
|
||||
|
||||
export const entityResolutionService = new EntityResolutionService();
|
||||
|
||||
let batchService: EntityResolutionService | undefined;
|
||||
export function getBatchEntityResolutionService(): EntityResolutionService {
|
||||
if (!batchService) batchService = new EntityResolutionService({ n0dClient: createN0dClient() });
|
||||
return batchService;
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
/**
|
||||
* src/services/external-writeback-service.ts
|
||||
* =============================================
|
||||
* `0pt.write_external_record` -- "POST a validated, already-approved
|
||||
* structured record to a registered external-system adapter." Generic in its
|
||||
* core: this service has no Odoo/invoice/Protosell-specific logic anywhere
|
||||
* in it -- it resolves an `adapter_id` to a registered `WritebackAdapter`
|
||||
* (../adapters/writeback-adapter.ts) and delegates the actual write to it.
|
||||
* `../adapters/odoo-bills-adapter.ts`'s OdooBillsAdapter is ONE such
|
||||
* adapter, registered by a caller the same way any other adapter would be --
|
||||
* this service never imports it directly.
|
||||
*
|
||||
* WHY THIS EXISTS / WHERE IT SITS IN THE CHAIN: `act0/src/
|
||||
* approval-writeback-templates.ts`'s generic `0pt_document_approval_then_
|
||||
* external_write` Template chains a captured+extracted document
|
||||
* (0pt.document_ocr's real terminal event, `0pt.document_extracted`) through
|
||||
* an Approval Engine gate, THEN calls this capability. By the time
|
||||
* `writeExternalRecord` runs, a human has already approved the write -- this
|
||||
* service still re-checks CL0UD policy (per docs/PRODUCT_PLAN.md section
|
||||
* 4.1's hard rule that CL0UD is always re-consulted, the same rule
|
||||
* document-extraction-service.ts already follows -- confirmed by grepping
|
||||
* this repo's src/services/ for evaluatePolicy()/Cl0udClient usage; the
|
||||
* only other real service that does the same is iceberg-write-service.ts.
|
||||
* An earlier version of this comment also cited `quality-check-service.ts`
|
||||
* here; that file does not exist anywhere in this repo),
|
||||
* but does not itself re-verify the upstream Approval record (ACT0 owns
|
||||
* approvals; 0pt's own capability boundary is "did CL0UD say yes for THIS
|
||||
* call", same separation of concerns every other 0pt service in this repo
|
||||
* already draws).
|
||||
*
|
||||
* IDEMPOTENCY (per this task's own instruction not to regress the proven
|
||||
* pattern ported from v0-datahub's `lib/odoo/invoice-bridge.ts`
|
||||
* `processInboxToOdoo`): that original code CLAIMS a row (an atomic
|
||||
* Postgres `UPDATE ... WHERE odoo_move_id IS NULL AND odoo_state != 'posting'`)
|
||||
* BEFORE ever calling the external system, so two concurrent triggers (a
|
||||
* manual approve racing a cron, or a retried ACT0 step racing itself) cannot
|
||||
* both post the same record. This service ports that SAME "claim before you
|
||||
* call out" shape forward as a generic, adapter-agnostic in-memory claim,
|
||||
* keyed on `(adapter_id, idempotency_key)`:
|
||||
* - the claim check-and-set happens SYNCHRONOUSLY, before this function's
|
||||
* first `await` -- JavaScript's single-threaded run-to-first-await
|
||||
* semantics make this exactly as race-free, within one process, as the
|
||||
* original's atomic SQL UPDATE was across concurrent DB clients (see
|
||||
* `writeExternalRecord`'s own inline comment for exactly where the line
|
||||
* is drawn, and tests/p0/external-writeback-service.test.ts's own
|
||||
* concurrency test, which calls `writeExternalRecord` twice with the
|
||||
* same idempotency key via `Promise.all` and asserts the underlying
|
||||
* adapter's `write` is invoked exactly once).
|
||||
* - HONEST SCOPE: this in-memory claim is process-local -- it does not
|
||||
* survive a restart and does not protect a multi-instance deployment,
|
||||
* unlike the original's real Postgres row lock. An adapter with its own
|
||||
* persistent storage (OdooBillsAdapter) additionally refuses to
|
||||
* re-post a record whose payload already carries an `existing_external_id`
|
||||
* -- see WritebackAdapter.write's own doc comment for why this
|
||||
* second layer matters and odoo-bills-adapter.ts for exactly how it is
|
||||
* ported. A production deployment backing this service with real
|
||||
* persistent storage (same "pure core, thin adapter" doctrine every
|
||||
* other 0pt service in this repo already documents) should replace this
|
||||
* in-memory Map with a real claim row, at which point the SAME
|
||||
* synchronous-claim-before-any-await shape still applies, now backed by
|
||||
* a real transaction instead of a JS Map.
|
||||
*/
|
||||
import crypto from "crypto";
|
||||
import { createCl0udClient, type Cl0udClient } from "../clients/cl0ud-client";
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import type { WritebackAdapter } from "../adapters/writeback-adapter";
|
||||
import type { ExternalWriteRecord, ExternalWriteRequest } from "../types/external-writeback";
|
||||
|
||||
export interface ExternalWritebackServiceOptions {
|
||||
cl0udBaseUrl?: string;
|
||||
aud0BaseUrl?: string;
|
||||
adapters?: readonly WritebackAdapter[];
|
||||
}
|
||||
|
||||
export class ExternalWritebackService {
|
||||
private writes: Map<string, ExternalWriteRecord> = new Map();
|
||||
/** `${adapter_id}::${idempotency_key}` -> write_id. See this module's own doc comment's
|
||||
* "IDEMPOTENCY" section for exactly what this does and does not guarantee. */
|
||||
private claimedByIdempotencyKey: Map<string, string> = new Map();
|
||||
private adapters: Map<string, WritebackAdapter> = new Map();
|
||||
private cl0ud: Cl0udClient;
|
||||
private aud0: Aud0Client;
|
||||
|
||||
constructor(opts?: ExternalWritebackServiceOptions) {
|
||||
this.cl0ud = createCl0udClient(opts?.cl0udBaseUrl);
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
for (const adapter of opts?.adapters ?? []) {
|
||||
this.registerAdapter(adapter);
|
||||
}
|
||||
}
|
||||
|
||||
registerAdapter(adapter: WritebackAdapter): void {
|
||||
this.adapters.set(adapter.adapterId, adapter);
|
||||
}
|
||||
|
||||
getAdapter(adapterId: string): WritebackAdapter {
|
||||
const adapter = this.adapters.get(adapterId);
|
||||
if (!adapter) throw new Error(`ExternalWritebackService: no adapter registered for '${adapterId}'`);
|
||||
return adapter;
|
||||
}
|
||||
|
||||
async writeExternalRecord(request: ExternalWriteRequest): Promise<ExternalWriteRecord> {
|
||||
const adapter = this.adapters.get(request.adapter_id);
|
||||
if (!adapter) {
|
||||
throw new Error(`ExternalWritebackService: no adapter registered for '${request.adapter_id}'`);
|
||||
}
|
||||
if (!adapter.isConfigured()) {
|
||||
throw new Error(
|
||||
`ExternalWritebackService: adapter '${request.adapter_id}' is not configured — refusing to fabricate a write`,
|
||||
);
|
||||
}
|
||||
|
||||
// --- SYNCHRONOUS claim section: no `await` above this line, and none until the claim is
|
||||
// set below. This is what makes the claim race-free under this module's own doc comment's
|
||||
// "IDEMPOTENCY" doctrine -- see that comment for the full explanation and its own scope
|
||||
// caveat (process-local only).
|
||||
const claimKey = `${request.adapter_id}::${request.idempotency_key}`;
|
||||
const existingWriteId = this.claimedByIdempotencyKey.get(claimKey);
|
||||
if (existingWriteId) {
|
||||
const existing = this.writes.get(existingWriteId);
|
||||
if (existing && existing.status === "written") {
|
||||
return existing; // already posted -- idempotent no-op, matches invoice-bridge.ts's own
|
||||
// "Redan bokförd i Odoo... hoppar över" early return.
|
||||
}
|
||||
if (existing && existing.status === "claimed") {
|
||||
// A write for this exact idempotency key is already in flight (or, in this in-memory
|
||||
// reference implementation, was left 'claimed' by a caller that never awaited the
|
||||
// result) -- do not race a second POST to the external system. Matches
|
||||
// invoice-bridge.ts's own "Redan under postning eller bokförd — hoppar över" claim-
|
||||
// conflict response.
|
||||
return { ...existing, status: "skipped_duplicate" };
|
||||
}
|
||||
// existing.status === 'failed' (or the record is missing for any other reason): fall
|
||||
// through and re-claim -- a failed write is safe and expected to retry, matching
|
||||
// invoice-bridge.ts's own "Release the 'posting' claim... can be retried later" behavior.
|
||||
}
|
||||
|
||||
const writeId = `ewr_${crypto.randomUUID()}`;
|
||||
const now = new Date().toISOString();
|
||||
let record: ExternalWriteRecord = {
|
||||
write_id: writeId,
|
||||
adapter_id: request.adapter_id,
|
||||
idempotency_key: request.idempotency_key,
|
||||
record_type: request.record_type,
|
||||
status: "claimed",
|
||||
tenant_id: request.tenant_id,
|
||||
project_id: request.project_id,
|
||||
actor_id: request.actor_id,
|
||||
correlation_id: request.correlation_id,
|
||||
created_at: now,
|
||||
};
|
||||
this.writes.set(writeId, record);
|
||||
this.claimedByIdempotencyKey.set(claimKey, writeId);
|
||||
// --- End synchronous claim section. Every branch below is async and safe to interleave. ---
|
||||
|
||||
try {
|
||||
const policy = await this.cl0ud.evaluatePolicy({
|
||||
tenant_id: request.tenant_id,
|
||||
project_id: request.project_id,
|
||||
actor_id: request.actor_id,
|
||||
capability_id: "0pt.write_external_record",
|
||||
resource_type: "external_write",
|
||||
resource_id: writeId,
|
||||
risk_level: "L4",
|
||||
correlation_id: request.correlation_id,
|
||||
});
|
||||
if (policy.decision === "deny") {
|
||||
throw new Error(`Policy denied: ${policy.reason}`);
|
||||
}
|
||||
if (policy.decision === "approval_required") {
|
||||
// Per this service's own module doc comment: the ACT0 Approval Engine gate that
|
||||
// precedes this capability in the approval-writeback-templates.ts chain is a SEPARATE
|
||||
// mechanism from CL0UD's own policy decision here -- if CL0UD independently still says
|
||||
// approval_required for this specific call, this service does not silently bypass it
|
||||
// (same "never lie about what happened" doctrine every other 0pt service in this repo
|
||||
// follows for a denied/approval_required decision).
|
||||
throw new Error(`Policy requires approval: ${policy.reason}`);
|
||||
}
|
||||
|
||||
const result = await adapter.write(
|
||||
{ recordType: request.record_type, payload: request.payload, idempotencyKey: request.idempotency_key },
|
||||
{
|
||||
tenant_id: request.tenant_id,
|
||||
project_id: request.project_id,
|
||||
actor_id: request.actor_id,
|
||||
correlation_id: request.correlation_id,
|
||||
},
|
||||
);
|
||||
if (!result.ok) {
|
||||
throw new Error(result.error ?? "adapter_write_failed");
|
||||
}
|
||||
|
||||
record = { ...record, status: "written", external_id: result.externalId, updated_at: new Date().toISOString() };
|
||||
this.writes.set(writeId, record);
|
||||
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_external_record_written_${writeId}`,
|
||||
event_type: "0pt.external_record_written",
|
||||
tenant_id: request.tenant_id,
|
||||
project_id: request.project_id,
|
||||
actor_id: request.actor_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.write_external_record",
|
||||
resource_type: "external_write",
|
||||
resource_id: writeId,
|
||||
risk_level: "L4",
|
||||
payload: {
|
||||
adapter_id: request.adapter_id,
|
||||
record_type: request.record_type,
|
||||
idempotency_key: request.idempotency_key,
|
||||
external_id: result.externalId,
|
||||
},
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: request.correlation_id,
|
||||
});
|
||||
|
||||
await this.aud0.ingestLineageEvent({
|
||||
lineage_event_id: `lineage_0pt_external_write_${writeId}`,
|
||||
tenant_id: request.tenant_id,
|
||||
project_id: request.project_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.write_external_record",
|
||||
// `idempotency_key` is, by convention (not enforced by this generic service), the
|
||||
// upstream record's stable id -- e.g. a document_extraction_id when this write was
|
||||
// triggered by act0's document_approval_then_external_write template.
|
||||
source_type: "document_extraction",
|
||||
source_id: request.idempotency_key,
|
||||
target_type: "external_record",
|
||||
target_id: result.externalId ?? writeId,
|
||||
lineage_type: "produced",
|
||||
metadata: { adapter_id: request.adapter_id, record_type: request.record_type },
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: request.correlation_id,
|
||||
});
|
||||
|
||||
return record;
|
||||
} catch (e: unknown) {
|
||||
const message = e instanceof Error ? e.message : String(e);
|
||||
record = { ...record, status: "failed", error: message, updated_at: new Date().toISOString() };
|
||||
this.writes.set(writeId, record);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async getWrite(writeId: string): Promise<ExternalWriteRecord> {
|
||||
const record = this.writes.get(writeId);
|
||||
if (!record) throw new Error(`External write ${writeId} not found`);
|
||||
return record;
|
||||
}
|
||||
|
||||
async findByIdempotencyKey(adapterId: string, idempotencyKey: string): Promise<ExternalWriteRecord | undefined> {
|
||||
const writeId = this.claimedByIdempotencyKey.get(`${adapterId}::${idempotencyKey}`);
|
||||
return writeId ? this.writes.get(writeId) : undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Shared singleton, same "cross-route in-memory state" convention document-capture-service.ts's
|
||||
* own header documents (this service's writes/claims must be visible across whatever API routes
|
||||
* call writeExternalRecord vs getWrite) -- no adapters pre-registered; a caller (an API route,
|
||||
* a test, or a future 0pt/api/documents/writeback route) registers whichever adapters it needs
|
||||
* via registerAdapter before calling writeExternalRecord. */
|
||||
export const externalWritebackService = new ExternalWritebackService();
|
||||
@@ -0,0 +1,342 @@
|
||||
import type { DatasetIcebergMapping } from "../types/dataset-iceberg-mapping";
|
||||
import { createCl0udClient, type Cl0udClient } from "../clients/cl0ud-client";
|
||||
import { createN0dClient, type N0dClient } from "../clients/n0d-client";
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import {
|
||||
createSt0reLakehouseClient,
|
||||
type St0reLakehouseClient,
|
||||
type IcebergTableFormat,
|
||||
type TableSnapshotType,
|
||||
} from "../clients/st0re-lakehouse-client";
|
||||
import { DatasetService } from "./dataset-service";
|
||||
import { DatasetIcebergMappingService } from "./dataset-iceberg-mapping-service";
|
||||
|
||||
/**
|
||||
* IcebergWriteService -- 0pt's write-path entrypoint for Iceberg tables.
|
||||
*
|
||||
* This is the concrete implementation of docs/ICEBERG_WRITE_BOUNDARY.md /
|
||||
* docs/11-lakehouse-write-plan.md items 1, 3, 6, 7, 8: 0pt is the canonical
|
||||
* writer of Iceberg tables, and it writes THROUGH ST0RE's table registry
|
||||
* (`St0reLakehouseClient`) rather than persisting any lakehouse table,
|
||||
* catalog, or snapshot state itself. This service holds NO table/snapshot
|
||||
* state of its own -- only `DatasetIcebergMappingService`'s lightweight
|
||||
* dataset-to-table-id pointer, which is a reference, not a copy.
|
||||
*
|
||||
* Two entry points:
|
||||
* - `mapDatasetToIcebergTable` -- registers a NEW Iceberg table in ST0RE for
|
||||
* a dataset that does not have one yet, then records the mapping. This is
|
||||
* the "first test dataset written to Iceberg through ST0RE's table
|
||||
* registry" proof path (see tests/p0/iceberg-write-service.test.ts).
|
||||
* - `requestIcebergWrite` -- registers a new snapshot (an actual write) for
|
||||
* an already-mapped dataset, optionally routed through an N0D job when the
|
||||
* write is heavy (item 7: 0pt REQUESTS N0D jobs for heavy writes/
|
||||
* compactions, it does not execute them itself, and it never owns the
|
||||
* result of that job beyond the ST0RE-registered snapshot ref).
|
||||
*
|
||||
* Byte-level writes (actually producing Parquet/Iceberg data files) are OUT
|
||||
* OF SCOPE for this pass, same honest boundary ST0RE itself documents for
|
||||
* its own snapshot routes ("ST0RE doesn't itself write the Iceberg
|
||||
* metadata.json... this registry tracks the pointer/version history for
|
||||
* whatever the caller's own catalog write produced" --
|
||||
* sax3l/st0re src/api/routes/lakehouse.ts). What this service proves is the
|
||||
* OWNERSHIP path -- that 0pt requests the write and registers its result
|
||||
* through ST0RE, never anywhere else.
|
||||
*/
|
||||
export interface MapDatasetToIcebergTableInput {
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
dataset_id: string;
|
||||
dataset_iceberg_mapping_id: string;
|
||||
namespace: string;
|
||||
table_name: string;
|
||||
storage_uri: string;
|
||||
catalog_ref: string;
|
||||
table_format?: IcebergTableFormat;
|
||||
actor_id: string;
|
||||
correlation_id: string;
|
||||
}
|
||||
|
||||
export interface MapDatasetToIcebergTableResult {
|
||||
mapping: DatasetIcebergMapping;
|
||||
iceberg_table_id: string;
|
||||
namespace_id: string;
|
||||
policy_decision_id: string;
|
||||
}
|
||||
|
||||
export interface RequestIcebergWriteInput {
|
||||
dataset_iceberg_mapping_id: string;
|
||||
/**
|
||||
* P014: optional so existing trusted in-process callers are unaffected,
|
||||
* but the HTTP route (app/api/0pt/lakehouse/mappings/[mappingId]/writes)
|
||||
* always supplies it now. When present, it must match the mapping's own
|
||||
* tenant_id or the write is refused as not-found -- see getMapping's
|
||||
* doc comment for the gap this closes (a foreign mapping_id used to let
|
||||
* any caller register a snapshot against another tenant's table).
|
||||
*/
|
||||
tenant_id?: string;
|
||||
actor_id: string;
|
||||
correlation_id: string;
|
||||
snapshot_type?: TableSnapshotType;
|
||||
schema_version: string;
|
||||
manifest_uri: string;
|
||||
record_count?: number;
|
||||
size_bytes?: number;
|
||||
/** True for large backfills/table maintenance -- routes through an N0D job request rather than 0pt executing in-process (docs/11-lakehouse-write-plan.md item 7). */
|
||||
heavy?: boolean;
|
||||
}
|
||||
|
||||
export interface RequestIcebergWriteResult {
|
||||
snapshot_id: string;
|
||||
iceberg_table_id: string;
|
||||
policy_decision_id: string;
|
||||
n0d_job_id?: string;
|
||||
}
|
||||
|
||||
export class IcebergWriteService {
|
||||
private cl0ud: Cl0udClient;
|
||||
private n0d: N0dClient;
|
||||
private st0reLakehouse: St0reLakehouseClient;
|
||||
private aud0: Aud0Client;
|
||||
private datasetService: DatasetService;
|
||||
private mappingService: DatasetIcebergMappingService;
|
||||
|
||||
constructor(opts?: {
|
||||
cl0udBaseUrl?: string;
|
||||
n0dBaseUrl?: string;
|
||||
st0reBaseUrl?: string;
|
||||
aud0BaseUrl?: string;
|
||||
datasetService?: DatasetService;
|
||||
mappingService?: DatasetIcebergMappingService;
|
||||
}) {
|
||||
this.cl0ud = createCl0udClient(opts?.cl0udBaseUrl);
|
||||
this.n0d = createN0dClient(opts?.n0dBaseUrl);
|
||||
this.st0reLakehouse = createSt0reLakehouseClient(opts?.st0reBaseUrl);
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
this.datasetService = opts?.datasetService ?? new DatasetService(opts?.aud0BaseUrl);
|
||||
this.mappingService = opts?.mappingService ?? new DatasetIcebergMappingService({ aud0BaseUrl: opts?.aud0BaseUrl, datasetService: this.datasetService });
|
||||
}
|
||||
|
||||
async mapDatasetToIcebergTable(input: MapDatasetToIcebergTableInput): Promise<MapDatasetToIcebergTableResult> {
|
||||
const dataset = await this.datasetService.getDataset(input.dataset_id); // throws if missing
|
||||
|
||||
// CL0UD policy boundary -- same L3 gate ST0RE's own table-registration
|
||||
// route applies (sax3l/st0re src/api/routes/lakehouse.ts,
|
||||
// 'st0re.lakehouse.write_table'), checked here too since 0pt is the
|
||||
// caller initiating the write, not just relying on ST0RE to gate it.
|
||||
const policy = await this.cl0ud.evaluatePolicy({
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
actor_id: input.actor_id,
|
||||
capability_id: "0pt.write_iceberg_table",
|
||||
resource_type: "dataset",
|
||||
resource_id: input.dataset_id,
|
||||
risk_level: "L3",
|
||||
correlation_id: input.correlation_id,
|
||||
});
|
||||
if (policy.decision === "deny") throw new Error(`Policy denied: ${policy.reason}`);
|
||||
if (policy.decision === "approval_required") throw new Error(`Policy requires approval: ${policy.reason}`);
|
||||
|
||||
const ctx = { tenantId: input.tenant_id, actorId: input.actor_id };
|
||||
|
||||
// Register (or reuse) the namespace THROUGH ST0RE's namespace registry
|
||||
// first, so the mapping can carry a real `namespace_id` (ST0RE's
|
||||
// LakehouseNamespace.namespaceId) rather than only the bare namespace
|
||||
// name string. See findOrRegisterNamespace's doc comment for why this is
|
||||
// a lookup-or-create, not a bare create.
|
||||
const namespace = await this.findOrRegisterNamespace(input.tenant_id, input.project_id, input.namespace, ctx);
|
||||
|
||||
// Register the table THROUGH ST0RE's table registry -- 0pt does not
|
||||
// create or track lakehouse table state on its own. NOTE: `datasetId` is
|
||||
// deliberately NOT sent -- it is a foreign key into ST0RE's OWN
|
||||
// dataset/artifact table (live-verified: sending 0pt's `dataset_id` here
|
||||
// 400s with a Postgres FK violation, since 0pt datasets are never rows in
|
||||
// ST0RE's dataset table). 0pt's dataset<->table linkage is carried
|
||||
// entirely by the DatasetIcebergMapping row created below.
|
||||
const table = await this.st0reLakehouse.registerTable(
|
||||
{
|
||||
tenantId: input.tenant_id,
|
||||
projectId: input.project_id,
|
||||
namespace: input.namespace,
|
||||
tableName: input.table_name,
|
||||
tableFormat: input.table_format ?? "iceberg", // ADR-0030: Iceberg is canonical; Delta/Hudi accepted only for bridge/import scenarios, never chosen by default here.
|
||||
storageUri: input.storage_uri,
|
||||
catalogRef: input.catalog_ref,
|
||||
ownerProduct: "0pt",
|
||||
createdBy: input.actor_id,
|
||||
},
|
||||
ctx
|
||||
);
|
||||
|
||||
const mapping = await this.mappingService.createMapping({
|
||||
dataset_iceberg_mapping_id: input.dataset_iceberg_mapping_id,
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
dataset_id: input.dataset_id,
|
||||
iceberg_table_id: table.tableId,
|
||||
namespace_id: namespace.namespaceId,
|
||||
namespace: input.namespace,
|
||||
table_name: input.table_name,
|
||||
mapping_status: "active",
|
||||
mapped_by: input.actor_id,
|
||||
correlation_id: input.correlation_id,
|
||||
});
|
||||
|
||||
return { mapping, iceberg_table_id: table.tableId, namespace_id: namespace.namespaceId, policy_decision_id: policy.decision_id };
|
||||
}
|
||||
|
||||
async requestIcebergWrite(input: RequestIcebergWriteInput): Promise<RequestIcebergWriteResult> {
|
||||
const mapping = await this.mappingService.getMapping(input.dataset_iceberg_mapping_id, input.tenant_id);
|
||||
if (mapping.mapping_status !== "active") {
|
||||
throw new Error(`Dataset Iceberg mapping ${input.dataset_iceberg_mapping_id} is not active (status=${mapping.mapping_status})`);
|
||||
}
|
||||
|
||||
const policy = await this.cl0ud.evaluatePolicy({
|
||||
tenant_id: mapping.tenant_id,
|
||||
project_id: mapping.project_id,
|
||||
actor_id: input.actor_id,
|
||||
capability_id: "0pt.write_iceberg_table",
|
||||
resource_type: "dataset",
|
||||
resource_id: mapping.dataset_id,
|
||||
risk_level: "L3",
|
||||
correlation_id: input.correlation_id,
|
||||
});
|
||||
if (policy.decision === "deny") throw new Error(`Policy denied: ${policy.reason}`);
|
||||
if (policy.decision === "approval_required") throw new Error(`Policy requires approval: ${policy.reason}`);
|
||||
|
||||
// 0pt.iceberg_write_requested -- fired when 0pt requests the write,
|
||||
// before the write lands (docs/11-lakehouse-write-plan.md event list).
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_iceberg_write_requested_${input.dataset_iceberg_mapping_id}_${Date.now()}`,
|
||||
event_type: "0pt.iceberg_write_requested",
|
||||
tenant_id: mapping.tenant_id,
|
||||
project_id: mapping.project_id,
|
||||
actor_id: input.actor_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.write_iceberg_table",
|
||||
resource_type: "iceberg_table",
|
||||
resource_id: mapping.iceberg_table_id,
|
||||
risk_level: "L3",
|
||||
payload: { dataset_id: mapping.dataset_id, dataset_iceberg_mapping_id: mapping.dataset_iceberg_mapping_id, heavy: input.heavy ?? false },
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: input.correlation_id,
|
||||
});
|
||||
|
||||
// Heavy writes/compactions are REQUESTED from N0D, never executed
|
||||
// in-process by 0pt (docs/11-lakehouse-write-plan.md item 7). N0D
|
||||
// executes; it does not own the schema or the decision of what to
|
||||
// write -- the snapshot registration below still goes through ST0RE
|
||||
// regardless of who performed the write.
|
||||
let n0dJobId: string | undefined;
|
||||
if (input.heavy) {
|
||||
const job = await this.n0d.createJob({
|
||||
tenant_id: mapping.tenant_id,
|
||||
project_id: mapping.project_id,
|
||||
job_type: "0pt.iceberg_write_job",
|
||||
capability_id: "0pt.write_iceberg_table",
|
||||
requested_by_app_id: "0pt",
|
||||
requested_by_actor_id: input.actor_id,
|
||||
input_dataset_ids: [mapping.dataset_id],
|
||||
output_artifact_type: "iceberg_table_snapshot",
|
||||
requires_backbone: true,
|
||||
requires_gpu: false,
|
||||
required_node_roles: ["compute_general"],
|
||||
resource_profile: "cpu_io_large",
|
||||
risk_level: "L3",
|
||||
correlation_id: input.correlation_id,
|
||||
});
|
||||
n0dJobId = job.job_id;
|
||||
}
|
||||
|
||||
// Register the snapshot THROUGH ST0RE's table registry -- the one and
|
||||
// only place 0pt persists that a write landed.
|
||||
const snapshot = await this.st0reLakehouse.createSnapshot(
|
||||
mapping.iceberg_table_id,
|
||||
{
|
||||
snapshotType: input.snapshot_type ?? "append",
|
||||
schemaVersion: input.schema_version,
|
||||
manifestUri: input.manifest_uri,
|
||||
createdByJobId: n0dJobId,
|
||||
createdByCapability: "0pt.write_iceberg_table",
|
||||
recordCount: input.record_count,
|
||||
sizeBytes: input.size_bytes,
|
||||
},
|
||||
{ tenantId: mapping.tenant_id, actorId: input.actor_id }
|
||||
);
|
||||
|
||||
// 0pt.iceberg_snapshot_created -- fired once the write lands and ST0RE
|
||||
// has registered a new snapshot, carrying table ref + snapshot id + row
|
||||
// count + dataset id for AUD0 to build source -> dataset -> snapshot
|
||||
// lineage.
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_iceberg_snapshot_created_${snapshot.snapshotId}`,
|
||||
event_type: "0pt.iceberg_snapshot_created",
|
||||
tenant_id: mapping.tenant_id,
|
||||
project_id: mapping.project_id,
|
||||
actor_id: input.actor_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.write_iceberg_table",
|
||||
resource_type: "iceberg_snapshot",
|
||||
resource_id: snapshot.snapshotId,
|
||||
risk_level: "L2",
|
||||
payload: {
|
||||
dataset_id: mapping.dataset_id,
|
||||
iceberg_table_id: mapping.iceberg_table_id,
|
||||
snapshot_id: snapshot.snapshotId,
|
||||
record_count: input.record_count,
|
||||
n0d_job_id: n0dJobId,
|
||||
},
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: input.correlation_id,
|
||||
});
|
||||
|
||||
await this.aud0.ingestLineageEvent({
|
||||
lineage_event_id: `lineage_0pt_iceberg_snapshot_${snapshot.snapshotId}`,
|
||||
tenant_id: mapping.tenant_id,
|
||||
project_id: mapping.project_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.write_iceberg_table",
|
||||
source_type: "dataset",
|
||||
source_id: mapping.dataset_id,
|
||||
target_type: "iceberg_snapshot",
|
||||
target_id: snapshot.snapshotId,
|
||||
lineage_type: "produced",
|
||||
metadata: { iceberg_table_id: mapping.iceberg_table_id, n0d_job_id: n0dJobId },
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: input.correlation_id,
|
||||
});
|
||||
|
||||
return {
|
||||
snapshot_id: snapshot.snapshotId,
|
||||
iceberg_table_id: mapping.iceberg_table_id,
|
||||
policy_decision_id: policy.decision_id,
|
||||
n0d_job_id: n0dJobId,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Looks up an existing ST0RE lakehouse namespace by (tenantId, projectId,
|
||||
* name) before creating one. ST0RE has no upsert route and enforces a real
|
||||
* unique constraint on that triple (live-verified 2026-07-13: a second
|
||||
* `registerNamespace` call for the same triple 400s with
|
||||
* `duplicate key value violates unique constraint
|
||||
* "st0re_lakehouse_namespaces_tenant_id_project_id_name_key"`), so a bare
|
||||
* "always create" call is not idempotent and would break every
|
||||
* `mapDatasetToIcebergTable` call after the first for a given namespace.
|
||||
*
|
||||
* ST0RE's list route does not reliably scope or filter results (see the
|
||||
* `listNamespaces` doc comment in st0re-lakehouse-client.ts, live-verified
|
||||
* finding #4) so the match is done here, client-side, against the full
|
||||
* returned list.
|
||||
*/
|
||||
private async findOrRegisterNamespace(
|
||||
tenantId: string,
|
||||
projectId: string,
|
||||
name: string,
|
||||
ctx: { tenantId: string; actorId: string }
|
||||
) {
|
||||
const existing = await this.st0reLakehouse.listNamespaces(ctx);
|
||||
const match = existing.find((n) => n.tenantId === tenantId && n.projectId === projectId && n.name === name);
|
||||
if (match) return match;
|
||||
return this.st0reLakehouse.registerNamespace({ tenantId, projectId, name, ownerProduct: "0pt" }, ctx);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { DatasetService } from "./dataset-service";
|
||||
import { DatasetIcebergMappingService } from "./dataset-iceberg-mapping-service";
|
||||
import { IcebergWriteService } from "./iceberg-write-service";
|
||||
|
||||
/**
|
||||
* Shared singleton instances for the lakehouse-mapping route group
|
||||
* (app/api/0pt/lakehouse/mappings/*). Previously each route file in that
|
||||
* group instantiated its own DatasetService/DatasetIcebergMappingService/
|
||||
* IcebergWriteService, so a mapping created via POST /mappings was invisible
|
||||
* to GET /mappings/{id} and to POST /mappings/{id}/writes within the same
|
||||
* server process (each held a distinct in-memory Map). Constructing all
|
||||
* three here once, wired together, and importing them everywhere fixes that
|
||||
* -- matching the in-memory-per-module convention this repo already uses
|
||||
* elsewhere (see document-capture-service.ts's own singleton note), just
|
||||
* applied consistently across this route group instead of per-file.
|
||||
*/
|
||||
export const datasetService = new DatasetService();
|
||||
export const mappingService = new DatasetIcebergMappingService({ datasetService });
|
||||
export const writeService = new IcebergWriteService({ datasetService, mappingService });
|
||||
@@ -0,0 +1,371 @@
|
||||
/**
|
||||
* src/services/pii-gate-service.ts
|
||||
* ==================================
|
||||
* 0pt.pii_* — the GDPR guardrail (fail-closed). Enforced by default wherever
|
||||
* a SourceQueryService is constructed; consulted by workers before they
|
||||
* persist anything for a personal-data source.
|
||||
*
|
||||
* Owns 0pt.pii_policy.schema.v1 and the in-memory identity registry that
|
||||
* backs the subject rights surfaces (access/erasure, GDPR art. 15/17) and
|
||||
* the retention worker (GDPR art. 5(1)(e)). Every decision is emitted to
|
||||
* AUD0 as risk L3 events so CL0UD gets an audit trail without polling 0pt.
|
||||
*
|
||||
* Directory domains known to publish personuppgifter (ratsit.se, …) are
|
||||
* pre-classified as personal: until a PiiPolicy with a recorded lawful
|
||||
* basis exists for them, the gate REJECTS storage/query of those sources.
|
||||
*/
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import type {
|
||||
PiiPolicy,
|
||||
CreatePiiPolicyInput,
|
||||
PiiIdentityKey,
|
||||
PiiErasureRequest,
|
||||
PiiAccessStatement,
|
||||
ProcessingPurpose,
|
||||
} from "../types/pii";
|
||||
import type { Source } from "../types/source";
|
||||
import { createAud0Client, type Aud0Client, type Aud0Event } from "../clients/aud0-client";
|
||||
|
||||
/** Domain publishes personuppgifter (directory/register listings). */
|
||||
const KNOWN_DIRECTORY_DOMAINS = [
|
||||
"ratsit.se",
|
||||
"merinfo.se",
|
||||
"hitta.se",
|
||||
"mrkoll.se",
|
||||
"eniro.se",
|
||||
"birthday.se",
|
||||
"upplysning.se",
|
||||
];
|
||||
|
||||
const DECISION_TEMPLATE = "PII source %s: %s";
|
||||
|
||||
export class PiiPolicyExistsError extends Error {}
|
||||
export class PiiLegalBasisError extends Error {}
|
||||
export class PiiMinimisationError extends Error {}
|
||||
export class PiiRetentionError extends Error {}
|
||||
export class PiiRobotsRateError extends Error {}
|
||||
export class PiiEntityLinkageError extends Error {}
|
||||
|
||||
export class PiiGateService {
|
||||
private policies: Map<string, PiiPolicy> = new Map();
|
||||
private registry: PiiIdentityKey[] = [];
|
||||
private aud0: Aud0Client;
|
||||
|
||||
constructor(opts?: { aud0BaseUrl?: string }) {
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
}
|
||||
|
||||
/** True if the source either self-flags or is a known directory domain. */
|
||||
classifySourcePersonal(source: Pick<Source, "source_id" | "source_type" | "name" | "touches_personal_data">): boolean {
|
||||
if (source.touches_personal_data === true) return true;
|
||||
if (source.source_type !== "web" && source.source_type !== "api") return false;
|
||||
const host = (source.name || "").toLowerCase().replace(/^https?:\/\//, "").replace(/\/.*$/, "");
|
||||
return KNOWN_DIRECTORY_DOMAINS.some((d) => host === d || host.endsWith("." + d));
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* Policies
|
||||
* ------------------------------------------------------------------ */
|
||||
|
||||
async createPiiPolicy(input: CreatePiiPolicyInput): Promise<PiiPolicy> {
|
||||
if (this.policies.has(input.source_id)) {
|
||||
throw new Error(`PiiPolicy already exists for source ${input.source_id}`);
|
||||
}
|
||||
// Hard validation — a policy is the lawful-basis gate, so it cannot be
|
||||
// written with a half-legal posture.
|
||||
if (!input.basis_ref.trim()) throw new Error("basis_ref is required (LIA id, consent id, contract clause or legal provision)");
|
||||
if (!Number.isFinite(input.retention_days) || input.retention_days < 0) {
|
||||
throw new Error("retention_days must be a non-negative integer (0 = fetch-and-drop)");
|
||||
}
|
||||
if (!Array.isArray(input.field_allowlist) || input.field_allowlist.length === 0) {
|
||||
throw new Error("field_allowlist must contain at least one field (data minimisation)");
|
||||
}
|
||||
|
||||
// Recorded decision (Simon, 2026-08-31): legitimate interest is accepted for
|
||||
// BUSINESS CONTACTS — personal data only in correspondence with a company/org
|
||||
// entity. Enforce that posture so "B2B-kontakt, företagskopplad" cannot be
|
||||
// bypassed by a lazily-written policy.
|
||||
const purpose: ProcessingPurpose = input.purpose ?? "business_contact";
|
||||
const entityLinkedOnly = input.entity_linked_only ?? input.legal_basis === "legitimate_interest";
|
||||
if (input.legal_basis === "legitimate_interest") {
|
||||
if (purpose !== "business_contact") {
|
||||
throw new Error("legitimate_interest is only accepted with purpose=business_contact (recorded decision 2026-08-31)");
|
||||
}
|
||||
if (!entityLinkedOnly) {
|
||||
throw new Error("legitimate_interest requires entity_linked_only=true — personal data must be bound to a company/org entity");
|
||||
}
|
||||
}
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const policy: PiiPolicy = {
|
||||
source_id: input.source_id,
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
touches_personal_data: true,
|
||||
legal_basis: input.legal_basis,
|
||||
purpose,
|
||||
entity_linked_only: entityLinkedOnly,
|
||||
basis_ref: input.basis_ref,
|
||||
retention_days: input.retention_days,
|
||||
field_allowlist: [...input.field_allowlist],
|
||||
pseudonymize_keys: input.pseudonymize_keys ?? true,
|
||||
robots_and_rate_confirmed: input.robots_and_rate_confirmed ?? false,
|
||||
created_at: now,
|
||||
updated_at: null,
|
||||
created_by: input.created_by ?? null,
|
||||
};
|
||||
this.policies.set(input.source_id, policy);
|
||||
|
||||
await this.safeEmit({
|
||||
event_id: `evt_0pt_pii_policy_created_${input.source_id}`,
|
||||
event_type: "0pt.pii_policy_created",
|
||||
capability_id: "0pt.pii_policy_created",
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
actor_id: input.created_by ?? "system",
|
||||
resource_type: "source_pii_policy",
|
||||
resource_id: input.source_id,
|
||||
risk_level: "L3",
|
||||
payload: {
|
||||
legal_basis: input.legal_basis,
|
||||
purpose,
|
||||
entity_linked_only: entityLinkedOnly,
|
||||
retention_days: input.retention_days,
|
||||
field_count: input.field_allowlist.length,
|
||||
},
|
||||
});
|
||||
return policy;
|
||||
}
|
||||
|
||||
async getPiiPolicy(sourceId: string): Promise<PiiPolicy | null> {
|
||||
return this.policies.get(sourceId) ?? null;
|
||||
}
|
||||
|
||||
async listPiiPolicies(tenantId?: string): Promise<PiiPolicy[]> {
|
||||
const all = Array.from(this.policies.values());
|
||||
return tenantId ? all.filter((p) => p.tenant_id === tenantId) : all;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* Enforcement (fail-closed)
|
||||
* ------------------------------------------------------------------ */
|
||||
|
||||
/**
|
||||
* Gate for a store/query of `fields` against `source`. Throws unless the
|
||||
* source is non-personal or a lawful, minimised, retention-bounded,
|
||||
* robots/rate-confirmed policy is on record.
|
||||
*/
|
||||
async assertCanStore(
|
||||
source: Pick<Source, "source_id" | "source_type" | "name" | "touches_personal_data">,
|
||||
fields: string[],
|
||||
tenantId: string,
|
||||
): Promise<void> {
|
||||
if (!this.classifySourcePersonal(source)) return;
|
||||
|
||||
const policy = this.policies.get(source.source_id);
|
||||
if (!policy) {
|
||||
throw new PiiLegalBasisError(
|
||||
DECISION_TEMPLATE.replace("%s", source.source_id).replace(
|
||||
"%s",
|
||||
`is a personal-data source with no PiiPolicy on record — a lawful basis (${KNOWN_DIRECTORY_DOMAINS.includes(source.name || "") ? "including robots.txt + rate limit confirmation" : "art. 6 GDPR"}) must be registered before any processing`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
if (policy.retention_days === 0 && fields.length > 0) {
|
||||
throw new PiiRetentionError(
|
||||
DECISION_TEMPLATE.replace("%s", source.source_id).replace(
|
||||
"%s",
|
||||
"policy forbids retention (retention_days = 0); no fields may be stored",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const disallowed = fields.filter((f) => !policy.field_allowlist.includes(f));
|
||||
if (disallowed.length > 0) {
|
||||
throw new PiiMinimisationError(
|
||||
DECISION_TEMPLATE.replace("%s", source.source_id).replace(
|
||||
"%s",
|
||||
`field(s) outside the allowlist: ${disallowed.join(", ")} (GDPR art. 5(1)(c) minimisation)`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
if ((source.source_type === "web" || source.source_type === "api") && !policy.robots_and_rate_confirmed) {
|
||||
throw new PiiRobotsRateError(
|
||||
DECISION_TEMPLATE.replace("%s", source.source_id).replace(
|
||||
"%s",
|
||||
"robots.txt / rate-limit not confirmed — required for web & api personal-data sources; re-check the source terms (do not hardcode)",
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* Subject rights: access (art. 15) & erasure (art. 17), retention (art. 5)
|
||||
* ------------------------------------------------------------------ */
|
||||
|
||||
/**
|
||||
* Register that one value was stored. Only hashes are persisted; the
|
||||
* plaintext identity never reaches the registry.
|
||||
*/
|
||||
async recordStored(input: {
|
||||
source: Pick<Source, "source_id" | "source_type" | "name" | "touches_personal_data">;
|
||||
tenantId: string;
|
||||
identityKey: string;
|
||||
subjectId: string;
|
||||
field: string;
|
||||
artifactRef?: string;
|
||||
/** Company/org entity the value is bound to (orgnr/rgs/LEI). REQUIRED when the policy is entity_linked_only. */
|
||||
entityId?: string;
|
||||
}): Promise<PiiIdentityKey> {
|
||||
await this.assertCanStore(input.source, [input.field], input.tenantId);
|
||||
const policy = this.policies.get(input.source.source_id)!;
|
||||
|
||||
if (policy.entity_linked_only && !input.entityId) {
|
||||
throw new PiiEntityLinkageError(
|
||||
DECISION_TEMPLATE.replace("%s", input.source.source_id).replace(
|
||||
"%s",
|
||||
"policy requires entity linkage (business contact, recorded decision 2026-08-31) — pass entityId (orgnr/rgs/LEI); personal data may only be stored in correspondence with a company/org entity",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const key: PiiIdentityKey = {
|
||||
key_hash: hash256(`${sourceIdSalt(input.source.source_id)}:${input.identityKey}`),
|
||||
subject_hash: hash256(input.subjectId),
|
||||
entity_hash: input.entityId ? hash256(normaliseEntity(input.entityId)) : null,
|
||||
source_id: input.source.source_id,
|
||||
tenant_id: input.tenantId,
|
||||
field: input.field,
|
||||
artifact_ref: input.artifactRef ?? null,
|
||||
stored_at: new Date(now).toISOString(),
|
||||
expires_at: new Date(now + policy.retention_days * 86_400_000).toISOString(),
|
||||
status: "active",
|
||||
};
|
||||
this.registry.push(key);
|
||||
|
||||
await this.safeEmit({
|
||||
event_id: `evt_0pt_pii_retained_${input.source.source_id}_${key.key_hash.slice(0, 12)}`,
|
||||
event_type: "0pt.pii_data_retained",
|
||||
capability_id: "0pt.pii_retain_identity",
|
||||
tenant_id: input.tenantId,
|
||||
project_id: "",
|
||||
actor_id: "system",
|
||||
resource_type: "pii_identity_key",
|
||||
resource_id: key.key_hash,
|
||||
risk_level: "L3",
|
||||
payload: {
|
||||
source_id: input.source.source_id,
|
||||
field: input.field,
|
||||
expires_at: key.expires_at,
|
||||
entity_linked: key.entity_hash !== null,
|
||||
},
|
||||
});
|
||||
return key;
|
||||
}
|
||||
|
||||
/** Right of access — everything currently held on a subject across sources. */
|
||||
async accessStatement(subjectHash: string): Promise<PiiAccessStatement> {
|
||||
const rows = this.registry.filter((r) => r.subject_hash === subjectHash && r.status === "active");
|
||||
return { subject_hash: subjectHash, rows, generated_at: new Date().toISOString() };
|
||||
}
|
||||
|
||||
/** Right to erasure — tombstone every held row for the subject. */
|
||||
async erase(request: PiiErasureRequest): Promise<number> {
|
||||
let count = 0;
|
||||
for (const row of this.registry) {
|
||||
if (row.subject_hash === request.subject_hash && row.status === "active") {
|
||||
row.status = "erased";
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
await this.safeEmit({
|
||||
event_id: `evt_0pt_pii_erased_${request.subject_hash.slice(0, 12)}_${Date.now()}`,
|
||||
event_type: "0pt.pii_erased",
|
||||
capability_id: "0pt.pii_erase",
|
||||
tenant_id: "",
|
||||
project_id: "",
|
||||
actor_id: request.actor_id ?? "system",
|
||||
resource_type: "pii_subject",
|
||||
resource_id: request.subject_hash,
|
||||
risk_level: "L3",
|
||||
payload: { reason: request.reason, row_count: count },
|
||||
});
|
||||
return count;
|
||||
}
|
||||
|
||||
/** Storage limitation — purge rows past their retention deadline. */
|
||||
async enforceRetention(now: Date = new Date()): Promise<{ purged: number }> {
|
||||
let purged = 0;
|
||||
const keep: PiiIdentityKey[] = [];
|
||||
for (const row of this.registry) {
|
||||
if (row.expires_at <= now.toISOString() && row.status === "active") {
|
||||
row.status = "purged";
|
||||
purged += 1;
|
||||
} else {
|
||||
keep.push(row);
|
||||
}
|
||||
}
|
||||
if (purged > 0) {
|
||||
await this.safeEmit({
|
||||
event_id: `evt_0pt_pii_retention_${now.getTime()}`,
|
||||
event_type: "0pt.pii_retention_purged",
|
||||
capability_id: "0pt.pii_enforce_retention",
|
||||
tenant_id: "",
|
||||
project_id: "",
|
||||
actor_id: "system",
|
||||
resource_type: "pii_identity_key",
|
||||
resource_id: now.toISOString(),
|
||||
risk_level: "L3",
|
||||
payload: { row_count: purged },
|
||||
});
|
||||
}
|
||||
return { purged };
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* Log hygiene — no personuppgifter in logs/metrics/traces
|
||||
* ------------------------------------------------------------------ */
|
||||
|
||||
/** Scrub personnummer, e-mail and Swedish phone numbers from a log line. */
|
||||
redact(line: string): string {
|
||||
return line
|
||||
.replace(/\b\d{8}[-–]\d{4}\b/g, "[PERSONNUMMER]")
|
||||
.replace(/\b\d{6}[-–]\d{4}\b/g, "[PERSONNUMMER]")
|
||||
.replace(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g, "[EMAIL]")
|
||||
.replace(/(?:\+46|0046|0)\s?[1-9][\d\s.-]{7,13}/g, "[TELEFON]");
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
private async safeEmit(event: Omit<Aud0Event, "app_id" | "created_at">): Promise<void> {
|
||||
const full: Aud0Event = {
|
||||
...event,
|
||||
app_id: "0pt",
|
||||
created_at: new Date().toISOString(),
|
||||
};
|
||||
try {
|
||||
await this.aud0.ingestEvent(full);
|
||||
} catch {
|
||||
// Audit is important but must never take down a compliance decision or
|
||||
// a retention pass. The caller's state change has already happened.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function sourceIdSalt(sourceId: string): string {
|
||||
return `0pt-pii-v1:${sourceId}`;
|
||||
}
|
||||
|
||||
/** Normalise an org/rgs nummer or LEI to canonical digits/letters before hashing. */
|
||||
function normaliseEntity(entityId: string): string {
|
||||
return entityId.replace(/[^0-9A-Za-z]/g, "").toUpperCase();
|
||||
}
|
||||
|
||||
/** sha256 hex digest — never the plaintext. */
|
||||
export function hash256(value: string): string {
|
||||
return createHash("sha256").update(value, "utf8").digest("hex");
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
/**
|
||||
* src/services/quality.ts
|
||||
* =========================
|
||||
* Shared singleton for the quality route group (app/api/0pt/quality/*),
|
||||
* matching src/services/source-registry.ts's precedent: constructing the
|
||||
* service once here (instead of `new QualityService()` per route module)
|
||||
* keeps rules/runs/issues registered via one route visible to another
|
||||
* within the same running process — the same "not even a shared
|
||||
* process-wide singleton" defect BEFORE_STATE.md §1 found and fixed for the
|
||||
* Source Registry would otherwise recur here.
|
||||
*
|
||||
* This is also the seam the Python engine's quality bridge
|
||||
* (services/api/workers/quality_bridge.py) calls through over HTTP —
|
||||
* see app/api/0pt/quality/bridge-run/route.ts.
|
||||
*/
|
||||
import { QualityService } from "../quality/quality-service";
|
||||
|
||||
export const qualityService = new QualityService();
|
||||
@@ -0,0 +1,33 @@
|
||||
import { createAud0Client } from "../clients/aud0-client";
|
||||
import { resolveDependencyBaseUrl } from "../platform/dependency-mode";
|
||||
import { SiaxError } from "../platform/errors";
|
||||
import { createSearchDefinitionRepository } from "../repositories/search-definition-repository";
|
||||
import { HttpOpenSearchAdapter } from "../search/opensearch-adapter";
|
||||
import { SearchService } from "../search/search-service";
|
||||
|
||||
let singleton: SearchService | undefined;
|
||||
|
||||
export function getSearchService(): SearchService {
|
||||
if (singleton) return singleton;
|
||||
const baseUrl = resolveDependencyBaseUrl("opensearch", process.env.OPENSEARCH_URL);
|
||||
if (!baseUrl) {
|
||||
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "opensearch", "OPENSEARCH_URL is required for the search HTTP surface");
|
||||
}
|
||||
singleton = new SearchService(
|
||||
new HttpOpenSearchAdapter({
|
||||
baseUrl,
|
||||
...(process.env.OPENSEARCH_BASIC_AUTH
|
||||
? { basicAuth: process.env.OPENSEARCH_BASIC_AUTH }
|
||||
: {}),
|
||||
}),
|
||||
{
|
||||
aud0Client: createAud0Client(),
|
||||
definitionRepository: createSearchDefinitionRepository(),
|
||||
},
|
||||
);
|
||||
return singleton;
|
||||
}
|
||||
|
||||
export function resetSearchServiceForTests(): void {
|
||||
singleton = undefined;
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
/**
|
||||
* src/services/source-query-service.ts
|
||||
* =======================================
|
||||
* 0pt.query_source — bounded, timeout-guarded synchronous query execution
|
||||
* against a registered Source (source_type = "database",
|
||||
* adapter_id = "fabric_onelake_sql" today; extensible via `ADAPTER_REGISTRY`).
|
||||
*
|
||||
* This is deliberately NOT a general-purpose query engine: it enforces a
|
||||
* max-rows and max-time ceiling on every call and is meant for small,
|
||||
* interactive/preview-style reads. Anything heavier (large exports, long
|
||||
* scans, scheduled pulls) must go through N0D as a job — see
|
||||
* docs/P0_QUERY_SOURCE_FLOW.md — the same "synchronous vs. N0D job" split
|
||||
* `quality-check-service.ts` uses for quality checks.
|
||||
*
|
||||
* Reuses the existing SourceService (src/services/source-service.ts) purely
|
||||
* through its public `getSource()` — this file does not modify that class or
|
||||
* the Source/CreateSourceInput types, which already carry the `adapter_id`
|
||||
* field this capability depends on.
|
||||
*/
|
||||
|
||||
import type { Source } from "../types/source";
|
||||
import { SourceService } from "./source-service";
|
||||
import { PiiGateService } from "./pii-gate-service";
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import { FabricOneLakeAdapter, type FabricOneLakeConnectionConfig, type AdapterQueryResult } from "../adapters/fabric-onelake-adapter";
|
||||
|
||||
/** Default bound applied when the caller does not specify one. */
|
||||
export const SOURCE_QUERY_DEFAULT_MAX_ROWS = 1000;
|
||||
/** Absolute ceiling — a caller cannot raise max_rows above this on the synchronous path. */
|
||||
export const SOURCE_QUERY_HARD_MAX_ROWS = 10_000;
|
||||
/** Default timeout applied when the caller does not specify one. */
|
||||
export const SOURCE_QUERY_DEFAULT_TIMEOUT_MS = 10_000;
|
||||
/** Absolute ceiling — a caller cannot raise timeout_ms above this on the synchronous path. */
|
||||
export const SOURCE_QUERY_HARD_MAX_TIMEOUT_MS = 30_000;
|
||||
|
||||
/** Adapter ids supported by the synchronous query path today. */
|
||||
export type SupportedQueryAdapterId = "fabric_onelake_sql";
|
||||
|
||||
export class SourceQueryTimeoutError extends Error {}
|
||||
export class SourceQueryUnsupportedError extends Error {}
|
||||
|
||||
export interface QuerySourceInput {
|
||||
source_id: string;
|
||||
query: string;
|
||||
params?: Record<string, unknown>;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
actor_id?: string;
|
||||
correlation_id?: string;
|
||||
/** Clamped to [1, SOURCE_QUERY_HARD_MAX_ROWS]. Defaults to SOURCE_QUERY_DEFAULT_MAX_ROWS. */
|
||||
max_rows?: number;
|
||||
/** Clamped to [1, SOURCE_QUERY_HARD_MAX_TIMEOUT_MS]. Defaults to SOURCE_QUERY_DEFAULT_TIMEOUT_MS. */
|
||||
timeout_ms?: number;
|
||||
/**
|
||||
* Connection config for the source's adapter. Never persisted by this
|
||||
* service and never defaulted from an env var — tenant-agnostic by
|
||||
* construction, same as the adapter itself. Today only the
|
||||
* "fabric_onelake_sql" shape is supported.
|
||||
*/
|
||||
connection_config: FabricOneLakeConnectionConfig;
|
||||
}
|
||||
|
||||
export interface QuerySourceResult {
|
||||
source_id: string;
|
||||
rows: Record<string, unknown>[];
|
||||
row_count: number;
|
||||
truncated: boolean;
|
||||
duration_ms: number;
|
||||
bounds: { max_rows: number; timeout_ms: number };
|
||||
}
|
||||
|
||||
function clamp(value: number, min: number, max: number): number {
|
||||
if (!Number.isFinite(value)) return min;
|
||||
return Math.min(Math.max(value, min), max);
|
||||
}
|
||||
|
||||
export class SourceQueryService {
|
||||
private sourceService: SourceService;
|
||||
private aud0: Aud0Client;
|
||||
/** GDPR guardrail — on by default, can be shared via opts so callers use one registry. */
|
||||
private piiGate: PiiGateService;
|
||||
|
||||
constructor(opts?: { sourceService?: SourceService; aud0BaseUrl?: string; piiGate?: PiiGateService }) {
|
||||
// Accepts an existing SourceService so callers (e.g. the API route) share
|
||||
// one in-memory registry instead of each constructing an empty one.
|
||||
this.sourceService = opts?.sourceService ?? new SourceService();
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
this.piiGate = opts?.piiGate ?? new PiiGateService({ aud0BaseUrl: opts?.aud0BaseUrl });
|
||||
}
|
||||
|
||||
async querySource(input: QuerySourceInput): Promise<QuerySourceResult> {
|
||||
const startedAt = Date.now();
|
||||
|
||||
// GDPR fail-closed: personal-data sources (self-flagged or known directory
|
||||
// domains such as ratsit.se) may not be processed without a PiiPolicy.
|
||||
// Runs BEFORE the capability/type gate so the lawful-basis error is the
|
||||
// one the caller sees for personal data.
|
||||
const preflight = await this.sourceService.getSource(input.source_id);
|
||||
await this.piiGate.assertCanStore(preflight, [], input.tenant_id);
|
||||
|
||||
const source = await this.getQueryableSource(input.source_id);
|
||||
|
||||
const maxRows = clamp(input.max_rows ?? SOURCE_QUERY_DEFAULT_MAX_ROWS, 1, SOURCE_QUERY_HARD_MAX_ROWS);
|
||||
const timeoutMs = clamp(input.timeout_ms ?? SOURCE_QUERY_DEFAULT_TIMEOUT_MS, 1, SOURCE_QUERY_HARD_MAX_TIMEOUT_MS);
|
||||
const correlationId = input.correlation_id ?? `corr_0pt_query_source_${input.source_id}_${startedAt}`;
|
||||
|
||||
const adapter = this.buildAdapter(source, input.connection_config);
|
||||
try {
|
||||
const { rows, durationMs } = await this.runWithTimeout(adapter.query(input.query, input.params), timeoutMs);
|
||||
|
||||
const truncated = rows.length > maxRows;
|
||||
const boundedRows = truncated ? rows.slice(0, maxRows) : rows;
|
||||
|
||||
await this.emitQueriedEvent({
|
||||
source,
|
||||
input,
|
||||
correlationId,
|
||||
rowCount: boundedRows.length,
|
||||
truncated,
|
||||
durationMs,
|
||||
ok: true,
|
||||
});
|
||||
|
||||
return {
|
||||
source_id: source.source_id,
|
||||
rows: boundedRows,
|
||||
row_count: boundedRows.length,
|
||||
truncated,
|
||||
duration_ms: durationMs,
|
||||
bounds: { max_rows: maxRows, timeout_ms: timeoutMs },
|
||||
};
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
await this.emitQueriedEvent({
|
||||
source,
|
||||
input,
|
||||
correlationId,
|
||||
rowCount: 0,
|
||||
truncated: false,
|
||||
durationMs: Date.now() - startedAt,
|
||||
ok: false,
|
||||
error: message,
|
||||
});
|
||||
throw err instanceof Error ? err : new Error(message);
|
||||
} finally {
|
||||
// Each call opens its own short-lived adapter/pool (see buildAdapter) —
|
||||
// always release it, whether the query succeeded, failed, or timed out.
|
||||
await adapter.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
private async getQueryableSource(sourceId: string): Promise<Source> {
|
||||
const source = await this.sourceService.getSource(sourceId);
|
||||
if (source.source_type !== "database") {
|
||||
throw new SourceQueryUnsupportedError(
|
||||
`Source ${sourceId} has source_type "${source.source_type}", not "database" — the synchronous 0pt.query_source path only ` +
|
||||
`supports database sources. Read non-database sources through a N0D job instead.`,
|
||||
);
|
||||
}
|
||||
if (source.adapter_id !== "fabric_onelake_sql") {
|
||||
throw new SourceQueryUnsupportedError(
|
||||
`Source ${sourceId} has adapter_id "${source.adapter_id ?? "(none)"}" — the synchronous 0pt.query_source path currently only ` +
|
||||
`supports adapter_id "fabric_onelake_sql". Register the source with that adapter_id, or run the read through N0D as a job instead.`,
|
||||
);
|
||||
}
|
||||
return source;
|
||||
}
|
||||
|
||||
private buildAdapter(source: Source, connectionConfig: FabricOneLakeConnectionConfig): FabricOneLakeAdapter {
|
||||
if (!connectionConfig) {
|
||||
throw new Error(`Source ${source.source_id}: connection_config is required to query a fabric_onelake_sql source`);
|
||||
}
|
||||
return new FabricOneLakeAdapter(connectionConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* Races the adapter query against a timeout. Note this bounds *waiting*
|
||||
* time on the caller's side — it does not cancel an in-flight TDS request
|
||||
* on timeout (the `mssql` driver has no cooperative cancellation hook used
|
||||
* here), so a timed-out query may still complete against the warehouse
|
||||
* after this call has already thrown. That is an accepted trade-off for a
|
||||
* synchronous, best-effort preview path; anything where that matters
|
||||
* belongs on the N0D job path instead.
|
||||
*/
|
||||
private async runWithTimeout<T>(promise: Promise<AdapterQueryResult<T>>, timeoutMs: number): Promise<AdapterQueryResult<T>> {
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const timeout = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => {
|
||||
reject(
|
||||
new SourceQueryTimeoutError(
|
||||
`source query exceeded the bounded synchronous timeout of ${timeoutMs}ms — run this query through N0D as a job instead of the synchronous 0pt.query_source path`,
|
||||
),
|
||||
);
|
||||
}, timeoutMs);
|
||||
});
|
||||
try {
|
||||
return await Promise.race([promise, timeout]);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
private async emitQueriedEvent(args: {
|
||||
source: Source;
|
||||
input: QuerySourceInput;
|
||||
correlationId: string;
|
||||
rowCount: number;
|
||||
truncated: boolean;
|
||||
durationMs: number;
|
||||
ok: boolean;
|
||||
error?: string;
|
||||
}): Promise<void> {
|
||||
const { source, input, correlationId, rowCount, truncated, durationMs, ok, error } = args;
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_source_queried_${source.source_id}_${Date.now()}`,
|
||||
event_type: "0pt.source_queried",
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
actor_id: input.actor_id ?? "system",
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.query_source",
|
||||
resource_type: "source",
|
||||
resource_id: source.source_id,
|
||||
risk_level: "L4",
|
||||
payload: {
|
||||
adapter_id: source.adapter_id,
|
||||
ok,
|
||||
row_count: rowCount,
|
||||
truncated,
|
||||
duration_ms: durationMs,
|
||||
query_preview: input.query.slice(0, 200),
|
||||
...(error ? { error } : {}),
|
||||
},
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: correlationId,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
import type {
|
||||
SourceRatePolicy,
|
||||
SourceRatePolicyStatus,
|
||||
CreateSourceRatePolicyInput,
|
||||
} from "../types/source-rate-policy";
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import { SourceService } from "./source-service";
|
||||
|
||||
interface SourceRatePolicyRecord {
|
||||
source_rate_policy_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
source_id: string;
|
||||
rate_limit_per_minute: number;
|
||||
crawl_budget: number;
|
||||
compliance_notes: string | null;
|
||||
domain: string | null;
|
||||
robots_ok: boolean | null;
|
||||
access: string | null;
|
||||
status: string;
|
||||
correlation_id: string | null;
|
||||
created_at: string;
|
||||
updated_at: string | null;
|
||||
created_by: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns 0pt.source_rate_policy.schema.v1 -- the genuine ownership repair:
|
||||
* source/crawl-domain rate limiting is 0pt's own compliance policy, moved
|
||||
* OUT of N0D's ComputePolicy.sourceRatePolicy (deprecated there per
|
||||
* ADR-0030's audit). See docs/ICEBERG_WRITE_BOUNDARY.md.
|
||||
*/
|
||||
export class SourceRatePolicyService {
|
||||
private policies: Map<string, SourceRatePolicyRecord> = new Map();
|
||||
private aud0: Aud0Client;
|
||||
private sourceService: SourceService;
|
||||
|
||||
constructor(opts?: { aud0BaseUrl?: string; sourceService?: SourceService }) {
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
this.sourceService = opts?.sourceService ?? new SourceService({ aud0BaseUrl: opts?.aud0BaseUrl });
|
||||
}
|
||||
|
||||
async createSourceRatePolicy(input: CreateSourceRatePolicyInput): Promise<SourceRatePolicy> {
|
||||
if (this.policies.has(input.source_rate_policy_id)) {
|
||||
throw new Error(`Source rate policy ${input.source_rate_policy_id} already exists`);
|
||||
}
|
||||
if (input.rate_limit_per_minute <= 0) {
|
||||
throw new Error("rate_limit_per_minute must be > 0");
|
||||
}
|
||||
if (input.crawl_budget <= 0) {
|
||||
throw new Error("crawl_budget must be > 0");
|
||||
}
|
||||
// Verify the source exists -- a rate policy with no source is meaningless.
|
||||
try {
|
||||
await this.sourceService.getSource(input.source_id);
|
||||
} catch {
|
||||
throw new Error(`Source ${input.source_id} not found`);
|
||||
}
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const record: SourceRatePolicyRecord = {
|
||||
source_rate_policy_id: input.source_rate_policy_id,
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
source_id: input.source_id,
|
||||
rate_limit_per_minute: input.rate_limit_per_minute,
|
||||
crawl_budget: input.crawl_budget,
|
||||
compliance_notes: input.compliance_notes ?? null,
|
||||
domain: input.domain ?? null,
|
||||
robots_ok: input.robots_ok ?? null,
|
||||
access: input.access ?? null,
|
||||
status: input.status ?? "active",
|
||||
correlation_id: input.correlation_id ?? null,
|
||||
created_at: now,
|
||||
updated_at: null,
|
||||
created_by: input.created_by ?? null,
|
||||
};
|
||||
this.policies.set(input.source_rate_policy_id, record);
|
||||
|
||||
// 0pt.source_rate_policy_created -- docs/11-lakehouse-write-plan.md event list,
|
||||
// docs/ICEBERG_WRITE_BOUNDARY.md. Gives AUD0/CL0UD an evented trail of 0pt's
|
||||
// crawl-compliance posture without polling 0pt's DB.
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_source_rate_policy_created_${input.source_rate_policy_id}`,
|
||||
event_type: "0pt.source_rate_policy_created",
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
actor_id: input.created_by ?? "system",
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.create_source_rate_policy",
|
||||
resource_type: "source_rate_policy",
|
||||
resource_id: input.source_rate_policy_id,
|
||||
risk_level: "L2",
|
||||
payload: {
|
||||
source_id: input.source_id,
|
||||
rate_limit_per_minute: input.rate_limit_per_minute,
|
||||
crawl_budget: input.crawl_budget,
|
||||
},
|
||||
created_at: now,
|
||||
correlation_id: input.correlation_id ?? "",
|
||||
});
|
||||
|
||||
return this.toPolicy(record);
|
||||
}
|
||||
|
||||
// P014: `tenantId` is optional so existing internal callers that already
|
||||
// resolved a policy from a tenant-scoped list (or are trusted, in-process
|
||||
// callers) are unaffected — but the HTTP route now always passes it, which
|
||||
// closes a real gap: this lookup used to be reachable by rate-policy ID
|
||||
// alone with NO tenant check anywhere, so any authenticated caller could
|
||||
// read another tenant's crawl-budget/rate-limit/compliance-notes record
|
||||
// just by guessing or enumerating IDs. A mismatched tenantId is reported
|
||||
// identically to "not found" (no existence oracle), matching the
|
||||
// convention SourceRegistryService.assertSourceTenant already established.
|
||||
async getSourceRatePolicy(sourceRatePolicyId: string, tenantId?: string): Promise<SourceRatePolicy> {
|
||||
const record = this.policies.get(sourceRatePolicyId);
|
||||
if (!record || (tenantId && record.tenant_id !== tenantId)) {
|
||||
throw new Error(`Source rate policy ${sourceRatePolicyId} not found`);
|
||||
}
|
||||
return this.toPolicy(record);
|
||||
}
|
||||
|
||||
async listSourceRatePolicies(filter?: {
|
||||
tenant_id?: string;
|
||||
project_id?: string;
|
||||
source_id?: string;
|
||||
status?: SourceRatePolicyStatus;
|
||||
}): Promise<SourceRatePolicy[]> {
|
||||
let results = Array.from(this.policies.values());
|
||||
if (filter?.tenant_id) results = results.filter((r) => r.tenant_id === filter.tenant_id);
|
||||
if (filter?.project_id) results = results.filter((r) => r.project_id === filter.project_id);
|
||||
if (filter?.source_id) results = results.filter((r) => r.source_id === filter.source_id);
|
||||
if (filter?.status) results = results.filter((r) => r.status === filter.status);
|
||||
return results.map((r) => this.toPolicy(r));
|
||||
}
|
||||
|
||||
private toPolicy(r: SourceRatePolicyRecord): SourceRatePolicy {
|
||||
return {
|
||||
source_rate_policy_id: r.source_rate_policy_id,
|
||||
tenant_id: r.tenant_id,
|
||||
project_id: r.project_id,
|
||||
source_id: r.source_id,
|
||||
rate_limit_per_minute: r.rate_limit_per_minute,
|
||||
crawl_budget: r.crawl_budget,
|
||||
compliance_notes: r.compliance_notes ?? undefined,
|
||||
domain: r.domain ?? undefined,
|
||||
robots_ok: r.robots_ok ?? undefined,
|
||||
access: (r.access as SourceRatePolicy["access"]) ?? undefined,
|
||||
status: r.status as SourceRatePolicyStatus,
|
||||
correlation_id: r.correlation_id ?? undefined,
|
||||
created_at: r.created_at,
|
||||
updated_at: r.updated_at ?? undefined,
|
||||
created_by: r.created_by ?? undefined,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,511 @@
|
||||
/**
|
||||
* src/services/source-registry-service.ts
|
||||
* ========================================
|
||||
* The P0 Source Registry capability surface (Master-DoD §10):
|
||||
*
|
||||
* source.register -> delegates to SourceService.createSource (owner of
|
||||
* 0pt.source.schema.v1) + optional rights/provenance
|
||||
* source.query -> listSources + rights/purpose filtering
|
||||
* source.describe -> full SourceDescriptor (base + rights + provenance
|
||||
* + health + freshness + rate policy)
|
||||
* source.health -> record + aggregate health observations
|
||||
* source.rate-policy -> already owned by SourceRatePolicyService; composed
|
||||
*
|
||||
* Tenant isolation: every read path REQUIRES a tenant_id and filters on it —
|
||||
* a source registered by tenant A is invisible to tenant B (tested in
|
||||
* tests/p0/source-registry.test.ts, incl. the cross-tenant attack case §50).
|
||||
* `getHealth`/`getFreshness` now take tenant_id directly and run the same
|
||||
* `assertSourceTenant` gate every other accessor does — they are exposed as
|
||||
* standalone HTTP routes now (app/api/0pt/sources/[sourceId]/health,
|
||||
* .../freshness), so they can no longer rely on an earlier caller having
|
||||
* already checked tenancy on their behalf.
|
||||
*
|
||||
* Purpose enforcement: `assertPurposeAllowed` is the single hook other 0pt
|
||||
* capabilities call before using a source for a given purpose (SearchService
|
||||
* and DatasetService both do). No rights record ⇒ fail closed (deny).
|
||||
*
|
||||
* Persistence: rights/provenance/health/freshness now live behind
|
||||
* SourceRegistryRepository (src/repositories/source-registry-repository.ts)
|
||||
* — real Postgres (`opt_source_rights`/`opt_source_provenance`/
|
||||
* `opt_source_health_checks`/`opt_source_freshness`,
|
||||
* sql/002_source_registry.sql) when DATABASE_URL is configured, in-memory
|
||||
* Maps otherwise. `resolveStoreMode` still fails production boot hard with
|
||||
* no DATABASE_URL (Master-DoD §3) — the difference from before is that a
|
||||
* configured production now actually WORKS (opt_sources/etc. exist), not
|
||||
* just fails safely.
|
||||
*/
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import { SiaxError } from "../platform/errors";
|
||||
import {
|
||||
createSourceRegistryRepository,
|
||||
type SourceRegistryRepository,
|
||||
} from "../repositories/source-registry-repository";
|
||||
import { SourceService, type SourceServiceOptions } from "./source-service";
|
||||
import { SourceRatePolicyService } from "./source-rate-policy-service";
|
||||
import type { CreateSourceInput, Source } from "../types/source";
|
||||
import type {
|
||||
SourceAuthorization,
|
||||
SourceClassification,
|
||||
SourceDescriptor,
|
||||
SourceFreshness,
|
||||
SourceHealth,
|
||||
SourceHealthCheck,
|
||||
SourceHealthStatus,
|
||||
SourceProvenance,
|
||||
SourceRights,
|
||||
SourceSchedule,
|
||||
} from "../types/source-registry";
|
||||
|
||||
export interface RegisterSourceInput extends CreateSourceInput {
|
||||
rights?: Omit<SourceRights, "source_id" | "tenant_id">;
|
||||
provenance?: Omit<SourceProvenance, "source_id" | "tenant_id" | "updated_at">;
|
||||
authorization?: Omit<SourceAuthorization, "source_id" | "tenant_id">;
|
||||
schedule?: Omit<SourceSchedule, "source_id" | "tenant_id" | "updated_at">;
|
||||
}
|
||||
|
||||
/**
|
||||
* Actor ids that may never appear as an authorization's `granted_by`.
|
||||
* P060 requires an accountable HUMAN decision behind a source going live;
|
||||
* a service identity granting itself access is exactly the rubber stamp the
|
||||
* gate exists to prevent.
|
||||
*/
|
||||
const NON_ACCOUNTABLE_ACTORS = /^(system|svc_|service:|robot|automation|anonymous)/i;
|
||||
|
||||
export class SourceRegistryService {
|
||||
private readonly repo: SourceRegistryRepository;
|
||||
private readonly aud0: Aud0Client;
|
||||
readonly sources: SourceService;
|
||||
readonly ratePolicies: SourceRatePolicyService;
|
||||
|
||||
constructor(opts?: {
|
||||
aud0BaseUrl?: string;
|
||||
/** Explicit DATABASE_URL override — mainly for tests. Falls back to resolveStoreMode's own env lookup when omitted. */
|
||||
databaseUrl?: string;
|
||||
sourceService?: SourceService;
|
||||
ratePolicyService?: SourceRatePolicyService;
|
||||
/** Inject a repository directly (tests) — bypasses resolveStoreMode entirely. */
|
||||
repository?: SourceRegistryRepository;
|
||||
}) {
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
this.repo = opts?.repository ?? createSourceRegistryRepository(opts?.databaseUrl);
|
||||
const sourceServiceOpts: SourceServiceOptions = {
|
||||
aud0BaseUrl: opts?.aud0BaseUrl,
|
||||
databaseUrl: opts?.databaseUrl,
|
||||
};
|
||||
this.sources = opts?.sourceService ?? new SourceService(sourceServiceOpts);
|
||||
this.ratePolicies =
|
||||
opts?.ratePolicyService ??
|
||||
new SourceRatePolicyService({ aud0BaseUrl: opts?.aud0BaseUrl, sourceService: this.sources });
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// source.register
|
||||
// -------------------------------------------------------------------------
|
||||
async registerSource(input: RegisterSourceInput): Promise<SourceDescriptor> {
|
||||
const { rights, provenance, authorization, schedule, ...base } = input;
|
||||
// P060: registration NEVER activates. A caller asking for status "active"
|
||||
// gets the source created "pending" and must go through activateSource,
|
||||
// which checks the classification. Silently honouring `status: "active"`
|
||||
// here would leave the gate wide open to its most obvious bypass.
|
||||
const requestedActive = base.status === "active";
|
||||
const source = await this.sources.createSource({
|
||||
...base,
|
||||
status: requestedActive ? "pending" : base.status,
|
||||
});
|
||||
if (rights) {
|
||||
await this.setRights({ ...rights, source_id: source.source_id, tenant_id: source.tenant_id });
|
||||
}
|
||||
if (provenance) {
|
||||
await this.setProvenance({
|
||||
...provenance,
|
||||
source_id: source.source_id,
|
||||
tenant_id: source.tenant_id,
|
||||
updated_at: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
if (authorization) {
|
||||
await this.setAuthorization({
|
||||
...authorization,
|
||||
source_id: source.source_id,
|
||||
tenant_id: source.tenant_id,
|
||||
});
|
||||
}
|
||||
if (schedule) {
|
||||
await this.setSchedule({
|
||||
...schedule,
|
||||
source_id: source.source_id,
|
||||
tenant_id: source.tenant_id,
|
||||
updated_at: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
// A registration that arrived fully classified AND asked to be active is
|
||||
// activated here — through the same gate, not around it.
|
||||
if (requestedActive) {
|
||||
await this.activateSource(source.source_id, source.tenant_id, base.created_by);
|
||||
}
|
||||
return this.describeSource(source.source_id, source.tenant_id);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// P060 authorization / schedule
|
||||
// -------------------------------------------------------------------------
|
||||
async setAuthorization(auth: SourceAuthorization): Promise<SourceAuthorization> {
|
||||
await this.assertSourceTenant(auth.source_id, auth.tenant_id);
|
||||
if (!auth.granted_by || NON_ACCOUNTABLE_ACTORS.test(auth.granted_by)) {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.source_registry",
|
||||
`authorization.granted_by must name an accountable actor, not "${auth.granted_by}"`,
|
||||
);
|
||||
}
|
||||
await this.repo.setAuthorization(auth);
|
||||
await this.emit("0pt.source_authorization_set", auth.tenant_id, auth.source_id, {
|
||||
basis: auth.basis,
|
||||
granted_by: auth.granted_by,
|
||||
expires_at: auth.expires_at,
|
||||
evidence_ref: auth.evidence_ref,
|
||||
});
|
||||
return auth;
|
||||
}
|
||||
|
||||
async getAuthorization(sourceId: string, tenantId: string): Promise<SourceAuthorization | undefined> {
|
||||
await this.assertSourceTenant(sourceId, tenantId);
|
||||
return this.repo.getAuthorization(sourceId);
|
||||
}
|
||||
|
||||
async setSchedule(schedule: SourceSchedule): Promise<SourceSchedule> {
|
||||
await this.assertSourceTenant(schedule.source_id, schedule.tenant_id);
|
||||
if (schedule.interval_seconds !== undefined && schedule.interval_seconds <= 0) {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.source_registry",
|
||||
"schedule.interval_seconds must be positive",
|
||||
);
|
||||
}
|
||||
await this.repo.setSchedule(schedule);
|
||||
await this.emit("0pt.source_schedule_set", schedule.tenant_id, schedule.source_id, {
|
||||
mode: schedule.mode,
|
||||
interval_seconds: schedule.interval_seconds,
|
||||
enabled: schedule.enabled,
|
||||
});
|
||||
return schedule;
|
||||
}
|
||||
|
||||
async getSchedule(sourceId: string, tenantId: string): Promise<SourceSchedule | undefined> {
|
||||
await this.assertSourceTenant(sourceId, tenantId);
|
||||
return this.repo.getSchedule(sourceId);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// P060 classification + activation gate
|
||||
// -------------------------------------------------------------------------
|
||||
/**
|
||||
* Is this source classified? A source is classified when ALL of the
|
||||
* following are on record:
|
||||
* - an accountable owner,
|
||||
* - an unexpired authorization from an accountable actor,
|
||||
* - rights: a license id AND at least one allowed purpose,
|
||||
* - provenance: how the data is obtained.
|
||||
*
|
||||
* Everything else in the registry (schedule, schema, checkpoint) describes
|
||||
* HOW a source runs, not WHETHER it may — those are deliberately not part
|
||||
* of the gate, and that choice is documented here rather than left implicit.
|
||||
*/
|
||||
async classifySource(
|
||||
sourceId: string,
|
||||
tenantId: string,
|
||||
now: Date = new Date(),
|
||||
): Promise<SourceClassification> {
|
||||
const source = await this.assertSourceTenant(sourceId, tenantId);
|
||||
const missing: string[] = [];
|
||||
|
||||
if (!source.owner_id) missing.push("owner_id is not set");
|
||||
|
||||
const auth = await this.repo.getAuthorization(sourceId);
|
||||
if (!auth) {
|
||||
missing.push("no authorization on record");
|
||||
} else {
|
||||
if (!auth.granted_by || NON_ACCOUNTABLE_ACTORS.test(auth.granted_by)) {
|
||||
missing.push(`authorization.granted_by "${auth.granted_by}" is not an accountable actor`);
|
||||
}
|
||||
if (auth.expires_at && new Date(auth.expires_at).getTime() <= now.getTime()) {
|
||||
missing.push(`authorization expired at ${auth.expires_at}`);
|
||||
}
|
||||
}
|
||||
|
||||
const rights = await this.repo.getRights(sourceId);
|
||||
if (!rights) {
|
||||
missing.push("no rights record (license/purpose) on record");
|
||||
} else {
|
||||
if (!rights.license_id) missing.push("rights.license_id is empty");
|
||||
if (rights.allowed_purposes.length === 0) missing.push("rights.allowed_purposes is empty");
|
||||
}
|
||||
|
||||
const provenance = await this.repo.getProvenance(sourceId);
|
||||
if (!provenance) missing.push("no provenance record (sourcing method) on record");
|
||||
|
||||
return { source_id: sourceId, classified: missing.length === 0, missing };
|
||||
}
|
||||
|
||||
/**
|
||||
* P060 DoD: an UNCLASSIFIED source cannot be activated. Fail-closed —
|
||||
* every missing fact is named at once, and nothing is written when the
|
||||
* check fails.
|
||||
*/
|
||||
async activateSource(sourceId: string, tenantId: string, actorId?: string): Promise<Source> {
|
||||
const classification = await this.classifySource(sourceId, tenantId);
|
||||
if (!classification.classified) {
|
||||
await this.emit("0pt.source_activation_denied", tenantId, sourceId, {
|
||||
missing: classification.missing,
|
||||
});
|
||||
throw new SiaxError(
|
||||
"AUTHORIZATION_DENIED",
|
||||
"0pt.source_registry",
|
||||
`source ${sourceId} is not classified and cannot be activated: ${classification.missing.join("; ")}`,
|
||||
);
|
||||
}
|
||||
const source = await this.sources.updateStatus(sourceId, "active", actorId);
|
||||
await this.emit("0pt.source_activated", tenantId, sourceId, { activated_by: actorId ?? null });
|
||||
return source;
|
||||
}
|
||||
|
||||
/** Take a source out of service. Always allowed — deactivating is never gated. */
|
||||
async deactivateSource(
|
||||
sourceId: string,
|
||||
tenantId: string,
|
||||
status: "paused" | "archived" | "deprecated" = "paused",
|
||||
actorId?: string,
|
||||
): Promise<Source> {
|
||||
await this.assertSourceTenant(sourceId, tenantId);
|
||||
const source = await this.sources.updateStatus(sourceId, status, actorId);
|
||||
await this.emit("0pt.source_deactivated", tenantId, sourceId, { status });
|
||||
return source;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// rights / license / purpose metadata
|
||||
// -------------------------------------------------------------------------
|
||||
async setRights(rights: SourceRights): Promise<SourceRights> {
|
||||
await this.assertSourceTenant(rights.source_id, rights.tenant_id);
|
||||
await this.repo.setRights(rights);
|
||||
await this.emit("0pt.source_rights_set", rights.tenant_id, rights.source_id, {
|
||||
license_id: rights.license_id,
|
||||
allowed_purposes: rights.allowed_purposes,
|
||||
redistribution: rights.redistribution,
|
||||
});
|
||||
return rights;
|
||||
}
|
||||
|
||||
async getRights(sourceId: string, tenantId: string): Promise<SourceRights | undefined> {
|
||||
await this.assertSourceTenant(sourceId, tenantId);
|
||||
return this.repo.getRights(sourceId);
|
||||
}
|
||||
|
||||
/**
|
||||
* The purpose gate other capabilities call before using a source. FAIL
|
||||
* CLOSED: no rights record, or a purpose outside the allowlist, throws
|
||||
* AUTHORIZATION_DENIED.
|
||||
*/
|
||||
async assertPurposeAllowed(sourceId: string, tenantId: string, purpose: string): Promise<SourceRights> {
|
||||
await this.assertSourceTenant(sourceId, tenantId);
|
||||
const rights = await this.repo.getRights(sourceId);
|
||||
if (!rights) {
|
||||
throw new SiaxError(
|
||||
"AUTHORIZATION_DENIED",
|
||||
"0pt.source_registry",
|
||||
`source ${sourceId} has no rights record — purpose "${purpose}" denied (fail closed)`,
|
||||
);
|
||||
}
|
||||
if (!rights.allowed_purposes.includes(purpose)) {
|
||||
throw new SiaxError(
|
||||
"AUTHORIZATION_DENIED",
|
||||
"0pt.source_registry",
|
||||
`purpose "${purpose}" is not in source ${sourceId}'s allowed purposes`,
|
||||
);
|
||||
}
|
||||
return rights;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// provenance / freshness
|
||||
// -------------------------------------------------------------------------
|
||||
async setProvenance(prov: SourceProvenance): Promise<SourceProvenance> {
|
||||
await this.assertSourceTenant(prov.source_id, prov.tenant_id);
|
||||
await this.repo.setProvenance(prov);
|
||||
await this.emit("0pt.source_provenance_set", prov.tenant_id, prov.source_id, {
|
||||
method: prov.method,
|
||||
authority: prov.authority,
|
||||
cadence: prov.cadence,
|
||||
});
|
||||
return prov;
|
||||
}
|
||||
|
||||
async getProvenance(sourceId: string, tenantId: string): Promise<SourceProvenance | undefined> {
|
||||
await this.assertSourceTenant(sourceId, tenantId);
|
||||
return this.repo.getProvenance(sourceId);
|
||||
}
|
||||
|
||||
/** Record a successful data observation (drives freshness). */
|
||||
async recordSuccess(sourceId: string, tenantId: string, atIso?: string): Promise<void> {
|
||||
await this.assertSourceTenant(sourceId, tenantId);
|
||||
await this.repo.recordSuccess(sourceId, tenantId, atIso ?? new Date().toISOString());
|
||||
}
|
||||
|
||||
async getFreshness(sourceId: string, tenantId: string, now: Date = new Date()): Promise<SourceFreshness> {
|
||||
await this.assertSourceTenant(sourceId, tenantId);
|
||||
const prov = await this.repo.getProvenance(sourceId);
|
||||
const last = await this.repo.getLastSuccess(sourceId);
|
||||
const sla = prov?.freshness_sla_hours;
|
||||
if (!last) {
|
||||
return { source_id: sourceId, freshness_sla_hours: sla, fresh: sla === undefined };
|
||||
}
|
||||
const ageHours = (now.getTime() - new Date(last).getTime()) / 3_600_000;
|
||||
return {
|
||||
source_id: sourceId,
|
||||
last_success_at: last,
|
||||
freshness_sla_hours: sla,
|
||||
age_hours: Math.round(ageHours * 100) / 100,
|
||||
fresh: sla === undefined ? true : ageHours <= sla,
|
||||
};
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// source.health
|
||||
// -------------------------------------------------------------------------
|
||||
async recordHealthCheck(check: SourceHealthCheck): Promise<SourceHealth> {
|
||||
await this.assertSourceTenant(check.source_id, check.tenant_id);
|
||||
await this.repo.appendHealthCheck(check);
|
||||
if (check.status === "up") {
|
||||
await this.recordSuccess(check.source_id, check.tenant_id, check.checked_at);
|
||||
}
|
||||
return this.getHealth(check.source_id, check.tenant_id);
|
||||
}
|
||||
|
||||
async getHealth(sourceId: string, tenantId: string): Promise<SourceHealth> {
|
||||
await this.assertSourceTenant(sourceId, tenantId);
|
||||
// Newest-first (repository contract) — index 0 is the most recent check.
|
||||
const list = await this.repo.listHealthChecks(sourceId);
|
||||
const last = list[0];
|
||||
let consecutiveFailures = 0;
|
||||
for (const check of list) {
|
||||
if (check.status === "down" || check.status === "degraded") consecutiveFailures++;
|
||||
else break;
|
||||
}
|
||||
const status: SourceHealthStatus = last?.status ?? "unknown";
|
||||
return { source_id: sourceId, status, last_check: last, consecutive_failures: consecutiveFailures };
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// source.query / source.describe
|
||||
// -------------------------------------------------------------------------
|
||||
async querySources(filter: {
|
||||
tenant_id: string;
|
||||
project_id?: string;
|
||||
source_type?: Source["source_type"];
|
||||
status?: Source["status"];
|
||||
/** Only sources whose rights allowlist includes this purpose. */
|
||||
purpose?: string;
|
||||
/** Only sources with at most this redistribution openness. */
|
||||
redistribution?: SourceRights["redistribution"];
|
||||
}): Promise<Source[]> {
|
||||
if (!filter.tenant_id) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.source_registry", "tenant_id is required on source.query");
|
||||
}
|
||||
let results = await this.sources.listSources({
|
||||
tenant_id: filter.tenant_id,
|
||||
project_id: filter.project_id,
|
||||
source_type: filter.source_type,
|
||||
status: filter.status,
|
||||
});
|
||||
if (filter.purpose !== undefined || filter.redistribution !== undefined) {
|
||||
const rightsEntries = await Promise.all(
|
||||
results.map(async (s) => [s.source_id, await this.repo.getRights(s.source_id)] as const),
|
||||
);
|
||||
const rightsBySource = new Map(rightsEntries);
|
||||
if (filter.purpose !== undefined) {
|
||||
results = results.filter((s) => {
|
||||
const rights = rightsBySource.get(s.source_id);
|
||||
return rights !== undefined && rights.allowed_purposes.includes(filter.purpose as string);
|
||||
});
|
||||
}
|
||||
if (filter.redistribution !== undefined) {
|
||||
results = results.filter(
|
||||
(s) => rightsBySource.get(s.source_id)?.redistribution === filter.redistribution,
|
||||
);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async describeSource(sourceId: string, tenantId: string): Promise<SourceDescriptor> {
|
||||
const source = await this.assertSourceTenant(sourceId, tenantId);
|
||||
const [policies, rights, provenance, health, freshness, authorization, schedule, classification] =
|
||||
await Promise.all([
|
||||
this.ratePolicies.listSourceRatePolicies({ tenant_id: tenantId, source_id: sourceId }),
|
||||
this.repo.getRights(sourceId),
|
||||
this.repo.getProvenance(sourceId),
|
||||
this.getHealth(sourceId, tenantId),
|
||||
this.getFreshness(sourceId, tenantId),
|
||||
this.repo.getAuthorization(sourceId),
|
||||
this.repo.getSchedule(sourceId),
|
||||
this.classifySource(sourceId, tenantId),
|
||||
]);
|
||||
return {
|
||||
source,
|
||||
rights,
|
||||
provenance,
|
||||
health,
|
||||
freshness,
|
||||
rate_policy: policies[0],
|
||||
authorization,
|
||||
schedule,
|
||||
classification,
|
||||
};
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// internals
|
||||
// -------------------------------------------------------------------------
|
||||
/**
|
||||
* Tenant gate for every read/write: resolves the source AND verifies the
|
||||
* caller's tenant owns it. Cross-tenant probes get NOT_FOUND (no existence
|
||||
* oracle), same 404-not-403 posture as ST0RE/N0D.
|
||||
*/
|
||||
private async assertSourceTenant(sourceId: string, tenantId: string): Promise<Source> {
|
||||
if (!tenantId) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.source_registry", "tenant_id is required");
|
||||
}
|
||||
let source: Source;
|
||||
try {
|
||||
source = await this.sources.getSource(sourceId);
|
||||
} catch {
|
||||
throw new SiaxError("NOT_FOUND", "0pt.source_registry", `source ${sourceId} not found`);
|
||||
}
|
||||
if (source.tenant_id !== tenantId) {
|
||||
throw new SiaxError("NOT_FOUND", "0pt.source_registry", `source ${sourceId} not found`);
|
||||
}
|
||||
return source;
|
||||
}
|
||||
|
||||
private async emit(
|
||||
eventType: string,
|
||||
tenantId: string,
|
||||
sourceId: string,
|
||||
payload: Record<string, unknown>,
|
||||
): Promise<void> {
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_${eventType}_${sourceId}_${Date.now()}`,
|
||||
event_type: eventType,
|
||||
tenant_id: tenantId,
|
||||
project_id: "p_platform",
|
||||
actor_id: "svc_0pt",
|
||||
app_id: "0pt",
|
||||
capability_id: eventType.replace(/^0pt\./, "0pt."),
|
||||
resource_type: "source",
|
||||
resource_id: sourceId,
|
||||
risk_level: "L2",
|
||||
payload,
|
||||
created_at: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
* src/services/source-registry.ts
|
||||
* =================================
|
||||
* Shared singleton instances for the source-registry route group
|
||||
* (app/api/0pt/sources/*), matching src/services/lakehouse-mapping-registry.ts's
|
||||
* precedent.
|
||||
*
|
||||
* BEFORE_STATE.md §1 flagged this as materially worse than plain
|
||||
* in-memory-per-module: each route file previously constructed its own
|
||||
* `new SourceService()` (and `new SourceRatePolicyService()`), so a source
|
||||
* registered via one route module was invisible to another within the same
|
||||
* running process — "not even a shared process-wide singleton". Once
|
||||
* DATABASE_URL is configured this stops mattering (state lives in Postgres,
|
||||
* genuinely shared across processes/deploys — see src/repositories/*), but
|
||||
* it still matters for the in-memory (no DATABASE_URL) dev/test mode, where
|
||||
* state must be shared within one process to be usable at all. Constructing
|
||||
* all three here once, wired together, and importing them everywhere fixes
|
||||
* both call sites this repo already had (sources/rate-policies/*,
|
||||
* sources/[sourceId]/query) plus every new route this pass adds.
|
||||
*/
|
||||
import { SourceService } from "./source-service";
|
||||
import { SourceRatePolicyService } from "./source-rate-policy-service";
|
||||
import { SourceRegistryService } from "./source-registry-service";
|
||||
|
||||
export const sourceService = new SourceService();
|
||||
export const sourceRatePolicyService = new SourceRatePolicyService({ sourceService });
|
||||
export const sourceRegistryService = new SourceRegistryService({
|
||||
sourceService,
|
||||
ratePolicyService: sourceRatePolicyService,
|
||||
});
|
||||
|
||||
/**
|
||||
* P060-P068 ingestion plane singletons. Same reasoning as above: the schema
|
||||
* registry, checkpoint store, ledger, quarantine and lineage projection all
|
||||
* share ONE repository so the in-memory dev/test mode is coherent within a
|
||||
* process, and one `resolveStoreMode` decision in production.
|
||||
*/
|
||||
import { createIngestionRepository } from "../repositories/ingestion-repository";
|
||||
import { SchemaRegistry } from "../schema/schema-registry";
|
||||
import { LineageStore } from "../lineage/lineage-store";
|
||||
import { IngestionRunner } from "../ingestion/ingestion-runner";
|
||||
|
||||
export const ingestionRepository = createIngestionRepository();
|
||||
export const schemaRegistry = new SchemaRegistry(ingestionRepository);
|
||||
export const lineageStore = new LineageStore(ingestionRepository);
|
||||
export const ingestionRunner = new IngestionRunner(ingestionRepository, schemaRegistry, lineageStore);
|
||||
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* src/services/source-service.ts
|
||||
* ===============================
|
||||
* Owns 0pt.source.schema.v1 — the per-tenant registry of upstream data
|
||||
* sources (databases, crawl domains, APIs, file drops). This is the registry
|
||||
* `SourceQueryService.querySource()` (src/services/source-query-service.ts)
|
||||
* and `SourceRatePolicyService` (src/services/source-rate-policy-service.ts)
|
||||
* both look a `source_id` up against — see docs/P0_QUERY_SOURCE_FLOW.md.
|
||||
*
|
||||
* Persistence: storage now lives behind SourceRepository
|
||||
* (src/repositories/source-repository.ts) — real Postgres (`opt_sources`,
|
||||
* sql/002_source_registry.sql) when DATABASE_URL is configured, an
|
||||
* in-memory Map otherwise (non-production only; resolveStoreMode fails hard
|
||||
* in production with no URL). This service still owns the business rules
|
||||
* (validation, tenant-agnostic CRUD orchestration) and the AUD0 emission —
|
||||
* only the storage moved.
|
||||
*
|
||||
* Callers that need a shared view across route modules should use the
|
||||
* shared singleton in src/services/source-registry.ts rather than
|
||||
* constructing a fresh `new SourceService()` per module — see that file's
|
||||
* doc comment for the "each route owned its own registry" history this
|
||||
* fixes.
|
||||
*/
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import {
|
||||
createSourceRepository,
|
||||
type SourceRepository,
|
||||
type SourceListFilter,
|
||||
} from "../repositories/source-repository";
|
||||
import type { CreateSourceInput, Source, SourceStatus, SourceType } from "../types/source";
|
||||
|
||||
export interface SourceServiceOptions {
|
||||
aud0BaseUrl?: string;
|
||||
/** Explicit DATABASE_URL override — mainly for tests pointing at a specific Postgres instance. Falls back to resolveStoreMode's own env lookup when omitted. */
|
||||
databaseUrl?: string;
|
||||
/** Inject a repository directly (tests) — bypasses resolveStoreMode entirely. */
|
||||
repository?: SourceRepository;
|
||||
}
|
||||
|
||||
export class SourceService {
|
||||
private readonly repo: SourceRepository;
|
||||
private readonly aud0: Aud0Client;
|
||||
|
||||
constructor(opts?: SourceServiceOptions) {
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
this.repo = opts?.repository ?? createSourceRepository(opts?.databaseUrl);
|
||||
}
|
||||
|
||||
async createSource(input: CreateSourceInput): Promise<Source> {
|
||||
const source = await this.repo.create(input);
|
||||
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_source_created_${source.source_id}`,
|
||||
event_type: "0pt.source_created",
|
||||
tenant_id: source.tenant_id,
|
||||
project_id: source.project_id,
|
||||
actor_id: source.created_by ?? "system",
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.create_source",
|
||||
resource_type: "source",
|
||||
resource_id: source.source_id,
|
||||
risk_level: "L2",
|
||||
payload: {
|
||||
source_type: source.source_type,
|
||||
name: source.name,
|
||||
adapter_id: source.adapter_id,
|
||||
access: source.access,
|
||||
},
|
||||
created_at: source.created_at,
|
||||
correlation_id: source.correlation_id,
|
||||
});
|
||||
|
||||
return source;
|
||||
}
|
||||
|
||||
async getSource(sourceId: string): Promise<Source> {
|
||||
return this.repo.get(sourceId);
|
||||
}
|
||||
|
||||
/**
|
||||
* P060: the ONLY status transition path. Callers do not get a generic
|
||||
* update method, because status is what the activation gate protects —
|
||||
* SourceRegistryService.activateSource is the only caller that moves a
|
||||
* source to "active", and it runs the classification check first.
|
||||
*/
|
||||
async updateStatus(sourceId: string, status: SourceStatus, actorId?: string): Promise<Source> {
|
||||
const source = await this.repo.updateStatus(sourceId, status);
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_source_status_${source.source_id}_${Date.now()}`,
|
||||
event_type: "0pt.source_status_changed",
|
||||
tenant_id: source.tenant_id,
|
||||
project_id: source.project_id,
|
||||
actor_id: actorId ?? source.owner_id ?? "system",
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.update_source_status",
|
||||
resource_type: "source",
|
||||
resource_id: source.source_id,
|
||||
risk_level: "L2",
|
||||
payload: { status: source.status, name: source.name },
|
||||
created_at: source.updated_at ?? new Date().toISOString(),
|
||||
correlation_id: source.correlation_id,
|
||||
});
|
||||
return source;
|
||||
}
|
||||
|
||||
async listSources(filter?: {
|
||||
tenant_id?: string;
|
||||
project_id?: string;
|
||||
source_type?: SourceType;
|
||||
status?: SourceStatus;
|
||||
}): Promise<Source[]> {
|
||||
return this.repo.list(filter as SourceListFilter | undefined);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user