fix(recovery): restore source removed by production documentation merge

This commit is contained in:
2026-11-10 03:42:10 +01:00
parent 585bdd4b44
commit 3d2fc6f913
957 changed files with 115754 additions and 0 deletions
+4
View File
@@ -0,0 +1,4 @@
/** Shared singleton for the canonical-fact routes (same pattern as source-registry.ts). */
import { CanonicalFactService } from "./canonical-fact-service";
export const canonicalFactService = new CanonicalFactService();
+165
View File
@@ -0,0 +1,165 @@
/**
* src/services/canonical-fact-service.ts
* ======================================
* 0pt's canonical structured-observation write/read surface (sql/022).
*
* Ownership boundary: 0pt stores normalized OBSERVED facts (what a source
* said, when, from which artifact). L0RE owns epistemic propositions and
* projections and must consume these rows through this API. Nothing here
* overwrites an earlier observation: the unique key includes source_ref +
* observed_at, so conflicting sources are preserved as separate rows.
*
* Idempotency: identical re-writes hit ON CONFLICT DO NOTHING and are
* reported as `duplicates`, never duplicated.
*/
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import { SiaxError } from "../platform/errors";
import {
createCanonicalFactRepository,
entityKey,
type CanonicalFactRepository,
} from "../repositories/canonical-fact-repository";
import type {
CanonicalEntity,
CanonicalEntityInput,
CanonicalFact,
CanonicalFactInput,
CanonicalQuery,
CanonicalRelationInput,
CanonicalWriteRequest,
CanonicalWriteResult,
FactValueType,
} from "../types/canonical-facts";
const FACT_VALUE_TYPES: FactValueType[] = ["string", "number", "boolean", "date", "json"];
function inferValueType(value: unknown): FactValueType {
if (typeof value === "string") return "string";
if (typeof value === "number") return "number";
if (typeof value === "boolean") return "boolean";
return "json";
}
function assertEntity(entity: CanonicalEntityInput, label: string): void {
if (!entity || !entity.entity_type || !entity.identity_namespace || !entity.identity_value) {
throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", `${label} requires entity_type, identity_namespace and identity_value`);
}
}
function assertFact(fact: CanonicalFactInput, index: number): void {
assertEntity(fact.entity, `facts[${index}].entity`);
if (!fact.predicate) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", `facts[${index}].predicate is required`);
if (!fact.source_ref) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", `facts[${index}].source_ref is required`);
if (!fact.extractor_version) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", `facts[${index}].extractor_version is required`);
if (!fact.observed_at || Number.isNaN(Date.parse(fact.observed_at))) {
throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", `facts[${index}].observed_at must be an ISO timestamp`);
}
if (fact.value === undefined) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", `facts[${index}].value is required`);
if (fact.value_type && !FACT_VALUE_TYPES.includes(fact.value_type)) {
throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", `facts[${index}].value_type is invalid`);
}
if (fact.confidence !== undefined && (fact.confidence < 0 || fact.confidence > 1)) {
throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", `facts[${index}].confidence must be between 0 and 1`);
}
}
function assertRelation(relation: CanonicalRelationInput, index: number): void {
assertEntity(relation.subject, `relations[${index}].subject`);
assertEntity(relation.object, `relations[${index}].object`);
if (!relation.predicate) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_relation", `relations[${index}].predicate is required`);
if (!relation.source_ref) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_relation", `relations[${index}].source_ref is required`);
if (!relation.extractor_version) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_relation", `relations[${index}].extractor_version is required`);
if (!relation.observed_at || Number.isNaN(Date.parse(relation.observed_at))) {
throw new SiaxError("INVALID_REQUEST", "0pt.canonical_relation", `relations[${index}].observed_at must be an ISO timestamp`);
}
}
export interface CanonicalFactServiceOptions {
repository?: CanonicalFactRepository;
aud0BaseUrl?: string;
databaseUrl?: string;
}
export class CanonicalFactService {
private readonly repo: CanonicalFactRepository;
private readonly aud0: Aud0Client;
constructor(opts?: CanonicalFactServiceOptions) {
this.repo = opts?.repository ?? createCanonicalFactRepository(opts?.databaseUrl);
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
}
async write(input: CanonicalWriteRequest): Promise<CanonicalWriteResult> {
if (!input.tenant_id) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", "tenant_id is required");
const facts = input.facts ?? [];
const relations = input.relations ?? [];
const explicit = input.entities ?? [];
if (facts.length === 0 && relations.length === 0 && explicit.length === 0) {
throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", "at least one entity, fact or relation is required");
}
facts.forEach(assertFact);
relations.forEach(assertRelation);
// Unique entity inputs from explicit + facts + relations.
const unique = new Map<string, CanonicalEntityInput>();
for (const e of [...explicit, ...facts.map((f) => f.entity), ...relations.flatMap((r) => [r.subject, r.object])]) {
assertEntity(e, "entity");
unique.set(entityKey(e), e);
}
const entities: CanonicalEntity[] = await this.repo.upsertEntities(input.tenant_id, [...unique.values()]);
const entityIds = new Map(entities.map((e) => [entityKey(e), e.entity_id]));
const normalizedFacts = facts.map((f) => ({ ...f, value_type: f.value_type ?? inferValueType(f.value) }));
const factResult = await this.repo.insertFacts(input.tenant_id, normalizedFacts, entityIds);
const relationResult = await this.repo.insertRelations(input.tenant_id, relations, entityIds);
// AUD0: counts and provenance refs only, never fact values (PII discipline).
try {
await this.aud0.ingestEvent({
event_id: `evt_0pt_canonical_facts_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
event_type: "0pt.canonical_facts_written",
tenant_id: input.tenant_id,
project_id: input.project_id ?? input.tenant_id,
actor_id: "0pt",
app_id: "0pt",
capability_id: "0pt.canonical_fact_write",
resource_type: "canonical_facts",
resource_id: input.tenant_id,
risk_level: "L2",
created_at: new Date().toISOString(),
payload: {
entities: entities.length,
facts_inserted: factResult.inserted,
facts_duplicates: factResult.duplicates,
relations_inserted: relationResult.inserted,
relations_duplicates: relationResult.duplicates,
source_refs: [...new Set([...facts.map((f) => f.source_ref), ...relations.map((r) => r.source_ref)])],
run_refs: [...new Set([...facts.map((f) => f.run_ref), ...relations.map((r) => r.run_ref)].filter(Boolean))],
},
});
} catch {
// AUD0 outage must not lose the canonical write; the caller sees counts.
}
return {
entities,
facts: normalizedFacts.map((f) => ({
fact_id: "",
entity_id: entityIds.get(entityKey(f.entity)) ?? "",
predicate: f.predicate,
})),
relations: relations.map((r) => ({
relation_id: "",
subject_id: entityIds.get(entityKey(r.subject)) ?? "",
predicate: r.predicate,
object_id: entityIds.get(entityKey(r.object)) ?? "",
})),
duplicates: factResult.duplicates + relationResult.duplicates,
};
}
async query(query: CanonicalQuery): Promise<CanonicalFact[]> {
if (!query.tenant_id) throw new SiaxError("INVALID_REQUEST", "0pt.canonical_fact", "tenant_id is required");
return this.repo.queryFacts({ ...query, limit: Math.min(query.limit ?? 100, 500) });
}
}
@@ -0,0 +1,183 @@
import type {
DatasetIcebergMapping,
DatasetIcebergMappingStatus,
CreateDatasetIcebergMappingInput,
} from "../types/dataset-iceberg-mapping";
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import { DatasetService } from "./dataset-service";
interface DatasetIcebergMappingRecord {
dataset_iceberg_mapping_id: string;
tenant_id: string;
project_id: string;
dataset_id: string;
iceberg_table_id: string;
namespace_id: string | null;
namespace: string | null;
table_name: string | null;
mapping_status: string;
mapped_at: string;
mapped_by: string | null;
correlation_id: string | null;
created_at: string;
updated_at: string | null;
}
/**
* Owns 0pt.dataset_iceberg_mapping.schema.v1 -- the pointer from a 0pt
* dataset to the Iceberg table it writes to. `iceberg_table_id` is always a
* REFERENCE into ST0RE's table registry (never a copy of ST0RE's table row);
* this service holds no table/catalog/snapshot state itself. See
* docs/ICEBERG_WRITE_BOUNDARY.md.
*
* This service does not call ST0RE directly -- it records a mapping to an
* `iceberg_table_id` that already exists in ST0RE's registry.
* `IcebergWriteService.mapDatasetToIcebergTable` (src/services/
* iceberg-write-service.ts) is the higher-level flow that registers a NEW
* table in ST0RE first and then calls this service to record the mapping --
* kept as two composable steps so a mapping can also be created against an
* already-registered table (e.g. one N0D or another 0pt job already set up).
*/
export class DatasetIcebergMappingService {
private mappings: Map<string, DatasetIcebergMappingRecord> = new Map();
private aud0: Aud0Client;
private datasetService: DatasetService;
constructor(opts?: { aud0BaseUrl?: string; datasetService?: DatasetService }) {
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.datasetService = opts?.datasetService ?? new DatasetService(opts?.aud0BaseUrl);
}
async createMapping(input: CreateDatasetIcebergMappingInput): Promise<DatasetIcebergMapping> {
if (this.mappings.has(input.dataset_iceberg_mapping_id)) {
throw new Error(`Dataset Iceberg mapping ${input.dataset_iceberg_mapping_id} already exists`);
}
if (!input.iceberg_table_id) {
throw new Error("iceberg_table_id is required (reference into ST0RE's table registry)");
}
// Verify the dataset exists -- a mapping with no dataset is meaningless.
try {
await this.datasetService.getDataset(input.dataset_id);
} catch {
throw new Error(`Dataset ${input.dataset_id} not found`);
}
const now = new Date().toISOString();
const record: DatasetIcebergMappingRecord = {
dataset_iceberg_mapping_id: input.dataset_iceberg_mapping_id,
tenant_id: input.tenant_id,
project_id: input.project_id,
dataset_id: input.dataset_id,
iceberg_table_id: input.iceberg_table_id,
namespace_id: input.namespace_id ?? null,
namespace: input.namespace ?? null,
table_name: input.table_name ?? null,
mapping_status: input.mapping_status ?? "active",
mapped_at: now,
mapped_by: input.mapped_by ?? null,
correlation_id: input.correlation_id ?? null,
created_at: now,
updated_at: null,
};
this.mappings.set(input.dataset_iceberg_mapping_id, record);
// 0pt.dataset_iceberg_table_mapped -- docs/11-lakehouse-write-plan.md event list,
// docs/ICEBERG_WRITE_BOUNDARY.md. Fired when the mapping is created/updated.
await this.aud0.ingestEvent({
event_id: `evt_0pt_dataset_iceberg_table_mapped_${input.dataset_iceberg_mapping_id}`,
event_type: "0pt.dataset_iceberg_table_mapped",
tenant_id: input.tenant_id,
project_id: input.project_id,
actor_id: input.mapped_by ?? "system",
app_id: "0pt",
capability_id: "0pt.map_dataset_iceberg_table",
resource_type: "dataset_iceberg_mapping",
resource_id: input.dataset_iceberg_mapping_id,
risk_level: "L2",
payload: {
dataset_id: input.dataset_id,
iceberg_table_id: input.iceberg_table_id,
namespace_id: input.namespace_id,
namespace: input.namespace,
table_name: input.table_name,
},
created_at: now,
correlation_id: input.correlation_id ?? "",
});
// Lineage: dataset -> iceberg table.
await this.aud0.ingestLineageEvent({
lineage_event_id: `lineage_0pt_dataset_iceberg_${input.dataset_iceberg_mapping_id}`,
tenant_id: input.tenant_id,
project_id: input.project_id,
app_id: "0pt",
capability_id: "0pt.map_dataset_iceberg_table",
source_type: "dataset",
source_id: input.dataset_id,
target_type: "iceberg_table",
target_id: input.iceberg_table_id,
lineage_type: "stored_as",
metadata: { dataset_iceberg_mapping_id: input.dataset_iceberg_mapping_id },
created_at: now,
correlation_id: input.correlation_id ?? "",
});
return this.toMapping(record);
}
// P014: `tenantId` is optional so trusted in-process callers (existing
// unit tests, and internal callers that already resolved the tenant some
// other way) are unaffected -- but both HTTP routes that reach this method
// now always pass it, closing a real gap: `getMapping` used to be
// reachable by mapping ID alone with NO tenant check anywhere, so any
// authenticated caller could read -- or, via requestIcebergWrite below,
// WRITE a new snapshot onto -- another tenant's dataset<->Iceberg-table
// mapping just by guessing/enumerating dataset_iceberg_mapping_id. A
// mismatched tenantId reports identically to "not found" (no existence
// oracle), matching SourceRegistryService.assertSourceTenant's convention.
async getMapping(mappingId: string, tenantId?: string): Promise<DatasetIcebergMapping> {
const record = this.mappings.get(mappingId);
if (!record || (tenantId && record.tenant_id !== tenantId)) {
throw new Error(`Dataset Iceberg mapping ${mappingId} not found`);
}
return this.toMapping(record);
}
async getMappingForDataset(datasetId: string): Promise<DatasetIcebergMapping | undefined> {
const record = Array.from(this.mappings.values()).find((r) => r.dataset_id === datasetId && r.mapping_status === "active");
return record ? this.toMapping(record) : undefined;
}
async listMappings(filter?: {
tenant_id?: string;
project_id?: string;
dataset_id?: string;
mapping_status?: DatasetIcebergMappingStatus;
}): Promise<DatasetIcebergMapping[]> {
let results = Array.from(this.mappings.values());
if (filter?.tenant_id) results = results.filter((r) => r.tenant_id === filter.tenant_id);
if (filter?.project_id) results = results.filter((r) => r.project_id === filter.project_id);
if (filter?.dataset_id) results = results.filter((r) => r.dataset_id === filter.dataset_id);
if (filter?.mapping_status) results = results.filter((r) => r.mapping_status === filter.mapping_status);
return results.map((r) => this.toMapping(r));
}
private toMapping(r: DatasetIcebergMappingRecord): DatasetIcebergMapping {
return {
dataset_iceberg_mapping_id: r.dataset_iceberg_mapping_id,
tenant_id: r.tenant_id,
project_id: r.project_id,
dataset_id: r.dataset_id,
iceberg_table_id: r.iceberg_table_id,
namespace_id: r.namespace_id ?? undefined,
namespace: r.namespace ?? undefined,
table_name: r.table_name ?? undefined,
mapping_status: r.mapping_status as DatasetIcebergMappingStatus,
mapped_at: r.mapped_at,
mapped_by: r.mapped_by ?? undefined,
correlation_id: r.correlation_id ?? undefined,
created_at: r.created_at,
updated_at: r.updated_at ?? undefined,
};
}
}
+80
View File
@@ -0,0 +1,80 @@
/**
* Owns 0pt.dataset.schema.v1 — the canonical registry of 0pt datasets.
*
* Storage is Postgres-backed through DatasetRepository whenever DATABASE_URL
* is configured. Non-production tests/local development may use the in-memory
* repository; production without a persistent store fails at construction via
* resolveStoreMode. Dataset content remains outside this registry.
*/
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import {
createDatasetRepository,
type DatasetListFilter,
type DatasetRepository,
} from "../repositories/dataset-repository";
import type { Dataset, DatasetStatus, DatasetType, CreateDatasetInput } from "../types/dataset";
export type { Dataset, DatasetStatus, DatasetType, CreateDatasetInput };
export type RegisterDatasetInput = CreateDatasetInput;
export interface DatasetServiceOptions {
aud0BaseUrl?: string;
databaseUrl?: string;
repository?: DatasetRepository;
}
function normalizeOptions(input?: string | DatasetServiceOptions): DatasetServiceOptions {
return typeof input === "string" ? { aud0BaseUrl: input } : input ?? {};
}
export class DatasetService {
private readonly repo: DatasetRepository;
private readonly aud0: Aud0Client;
/**
* A string argument remains supported as the historical aud0BaseUrl form;
* new callers should use the options object so repository/database injection
* is explicit and testable.
*/
constructor(input?: string | DatasetServiceOptions) {
const opts = normalizeOptions(input);
this.aud0 = createAud0Client(opts.aud0BaseUrl);
this.repo = opts.repository ?? createDatasetRepository(opts.databaseUrl);
}
async registerDataset(input: CreateDatasetInput): Promise<Dataset> {
const dataset = await this.repo.create(input);
await this.aud0.ingestEvent({
event_id: `evt_0pt_dataset_registered_${dataset.dataset_id}`,
event_type: "0pt.dataset_registered",
tenant_id: dataset.tenant_id,
project_id: dataset.project_id,
actor_id: dataset.created_by ?? "system",
app_id: "0pt",
capability_id: "0pt.register_dataset",
resource_type: "dataset",
resource_id: dataset.dataset_id,
risk_level: "L2",
payload: { name: dataset.name, dataset_type: dataset.dataset_type },
created_at: dataset.created_at,
correlation_id: dataset.correlation_id,
idempotency_key: `dataset-register:${dataset.tenant_id}:${dataset.dataset_id}`,
});
return dataset;
}
async getDataset(datasetId: string): Promise<Dataset> {
return this.repo.get(datasetId);
}
async listDatasets(filter?: {
tenant_id?: string;
project_id?: string;
dataset_type?: DatasetType;
status?: DatasetStatus;
}): Promise<Dataset[]> {
return this.repo.list(filter as DatasetListFilter | undefined);
}
}
+249
View File
@@ -0,0 +1,249 @@
/**
* Dataset version/query/export capability.
*
* Dataset metadata is canonical in DatasetService; immutable version metadata
* is persisted through DatasetVersionRepository; bytes remain canonical ST0RE
* artifacts. AUD0/OpenLineage record the evidence chain.
*/
import crypto from "crypto";
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import { createSt0reClient, type St0reClient } from "../clients/st0re-client";
import { LineageEmitter } from "../lineage/lineage-emitter";
import {
createDatasetVersionRepository,
type DatasetVersionRepository,
} from "../repositories/dataset-version-repository";
import { SiaxError } from "../platform/errors";
import { formatResourceRef, isResourceRef } from "../platform/resource-ref";
import type {
CreateDatasetVersionInput,
DatasetExport,
DatasetVersion,
} from "../types/dataset-version";
import { DatasetService } from "./dataset-service";
export type { CreateDatasetVersionInput, DatasetExport, DatasetVersion } from "../types/dataset-version";
export interface DatasetVersionServiceOptions {
aud0BaseUrl?: string;
st0reBaseUrl?: string;
databaseUrl?: string;
datasetService?: DatasetService;
repository?: DatasetVersionRepository;
}
export class DatasetVersionService {
private readonly repo: DatasetVersionRepository;
private readonly aud0: Aud0Client;
private readonly st0re: St0reClient;
private readonly lineage: LineageEmitter;
readonly datasets: DatasetService;
constructor(opts?: DatasetVersionServiceOptions) {
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.st0re = createSt0reClient(opts?.st0reBaseUrl);
this.lineage = new LineageEmitter(this.aud0);
this.datasets = opts?.datasetService ?? new DatasetService({
aud0BaseUrl: opts?.aud0BaseUrl,
databaseUrl: opts?.databaseUrl,
});
this.repo = opts?.repository ?? createDatasetVersionRepository(opts?.databaseUrl);
}
async createVersion(input: CreateDatasetVersionInput): Promise<DatasetVersion> {
await this.assertDatasetTenant(input.dataset_id, input.tenant_id);
if (!input.project_id?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "project_id is required");
if (!input.schema_version?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "schema_version is required");
if (!input.created_by?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "created_by is required");
if (input.record_count !== undefined && (!Number.isSafeInteger(input.record_count) || input.record_count < 0)) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "record_count must be a non-negative safe integer");
}
if (input.artifact_refs.length === 0) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "a dataset version needs at least one artifact ref");
}
for (const ref of input.artifact_refs) {
if (!isResourceRef(ref, "st0re", "artifact")) {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.dataset",
`artifact ref "${ref}" is not a canonical st0re:artifact:... reference (Master-DoD §5)`,
);
}
}
const version = await this.repo.create(input);
const versionRef = formatResourceRef({
product: "0pt",
resourceType: "dataset",
resourceId: input.dataset_id,
version: String(version.version),
});
await this.aud0.ingestEvent({
event_id: `evt_0pt_dataset_version_${input.dataset_id}_v${version.version}`,
event_type: "0pt.dataset_version_created",
tenant_id: input.tenant_id,
project_id: input.project_id,
actor_id: input.created_by,
app_id: "0pt",
capability_id: "0pt.dataset.version.v1",
resource_type: "dataset_version",
resource_id: `${input.dataset_id}@${version.version}`,
risk_level: "L2",
result: "success",
input_refs: version.artifact_refs,
output_refs: [versionRef],
payload: { schema_version: version.schema_version, record_count: version.record_count },
created_at: version.created_at,
occurred_at: version.created_at,
correlation_id: version.correlation_id,
idempotency_key: `dataset-version:${input.tenant_id}:${input.dataset_id}:v${version.version}`,
});
await this.lineage.emit({
tenant_id: input.tenant_id,
project_id: input.project_id,
capability_id: "0pt.dataset.version.v1",
run_id: `dataset-version:${input.dataset_id}:v${version.version}`,
event_type: "COMPLETE",
event_time: version.created_at,
inputs: version.artifact_refs.map((ref) => ({ name: ref, ref })),
outputs: [{ name: `dataset/${input.dataset_id}`, ref: versionRef }],
correlation_id: version.correlation_id,
});
return version;
}
async listVersions(datasetId: string, tenantId: string): Promise<DatasetVersion[]> {
await this.assertDatasetTenant(datasetId, tenantId);
return this.repo.list(datasetId, tenantId);
}
async getVersion(datasetId: string, tenantId: string, version: number): Promise<DatasetVersion> {
await this.assertDatasetTenant(datasetId, tenantId);
if (!Number.isSafeInteger(version) || version < 1) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "version must be a positive integer");
}
return this.repo.get(datasetId, tenantId, version);
}
async exportDataset(input: {
dataset_id: string;
tenant_id: string;
project_id: string;
version: number;
format: DatasetExport["format"];
object_location: { provider: string; bucket: string; key: string; region?: string };
size_bytes: number;
checksum_sha256: string;
actor_id: string;
correlation_id?: string;
}): Promise<DatasetExport> {
const version = await this.getVersion(input.dataset_id, input.tenant_id, input.version);
if (!(["parquet", "csv", "json"] as const).includes(input.format)) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "unsupported export format");
}
if (!Number.isSafeInteger(input.size_bytes) || input.size_bytes < 0) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "size_bytes must be a non-negative safe integer");
}
if (!/^[a-fA-F0-9]{64}$/.test(input.checksum_sha256)) {
throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "checksum_sha256 must be a 64-character hex digest");
}
const exportKey = `dsexp:${input.tenant_id}:${input.dataset_id}:v${input.version}:${input.format}:${input.checksum_sha256.toLowerCase()}`;
const exportId = `dsexp_${crypto.createHash("sha256").update(exportKey).digest("hex").slice(0, 32)}`;
const artifact = await this.st0re.createArtifact({
tenant_id: input.tenant_id,
project_id: input.project_id,
artifact_type: "dataset_export",
app_id: "0pt",
produced_by: "0pt.dataset.export.v1",
object_location: input.object_location,
content_type:
input.format === "parquet"
? "application/vnd.apache.parquet"
: input.format === "csv"
? "text/csv"
: "application/json",
size_bytes: input.size_bytes,
checksum_sha256: input.checksum_sha256.toLowerCase(),
pii_level: "none",
retention_policy: "standard",
data_residency: "eu",
risk_level: "L2",
correlation_id: input.correlation_id,
created_by: input.actor_id,
idempotency_key: exportKey,
});
const artifactRef = formatResourceRef({
product: "st0re",
resourceType: "artifact",
resourceId: artifact.artifact_id,
});
const createdAt = artifact.created_at ?? new Date().toISOString();
const record: DatasetExport = {
export_id: exportId,
dataset_id: input.dataset_id,
version: input.version,
tenant_id: input.tenant_id,
artifact_ref: artifactRef,
format: input.format,
created_at: createdAt,
};
const versionRef = formatResourceRef({
product: "0pt",
resourceType: "dataset",
resourceId: input.dataset_id,
version: String(version.version),
});
await this.aud0.ingestEvent({
event_id: `evt_0pt_dataset_export_${exportId}`,
event_type: "0pt.dataset_exported",
tenant_id: input.tenant_id,
project_id: input.project_id,
actor_id: input.actor_id,
app_id: "0pt",
capability_id: "0pt.dataset.export.v1",
resource_type: "dataset_export",
resource_id: exportId,
risk_level: "L3",
result: "success",
input_refs: [versionRef],
output_refs: [artifactRef],
payload: { format: input.format, size_bytes: input.size_bytes },
created_at: createdAt,
occurred_at: createdAt,
correlation_id: input.correlation_id,
idempotency_key: exportKey,
});
await this.lineage.emit({
tenant_id: input.tenant_id,
project_id: input.project_id,
capability_id: "0pt.dataset.export.v1",
run_id: `dataset-export:${exportId}`,
event_type: "COMPLETE",
event_time: createdAt,
inputs: [{ name: `dataset/${input.dataset_id}`, ref: versionRef }],
outputs: [{ name: artifactRef, ref: artifactRef }],
correlation_id: input.correlation_id,
});
return record;
}
private async assertDatasetTenant(datasetId: string, tenantId: string): Promise<void> {
if (!tenantId?.trim()) throw new SiaxError("INVALID_REQUEST", "0pt.dataset", "tenant_id is required");
let dataset;
try {
dataset = await this.datasets.getDataset(datasetId);
} catch {
throw new SiaxError("NOT_FOUND", "0pt.dataset", `dataset ${datasetId} not found`);
}
if (dataset.tenant_id !== tenantId) {
throw new SiaxError("NOT_FOUND", "0pt.dataset", `dataset ${datasetId} not found`);
}
}
}
+6
View File
@@ -0,0 +1,6 @@
/** Shared composition root for the dataset HTTP surface. */
import { DatasetService } from "./dataset-service";
import { DatasetVersionService } from "./dataset-version-service";
export const datasetService = new DatasetService();
export const datasetVersionService = new DatasetVersionService({ datasetService });
+273
View File
@@ -0,0 +1,273 @@
/**
* src/services/document-capture-service.ts
* =========================================
* Orchestrates the capture half of the 0pt.document_ocr capability:
* poll a customer-registered mailbox (Microsoft Graph, app-only) →
* download PDF attachments → persist the bytes as an artifact →
* enqueue extraction (an N0D job placeholder; a worker or the
* /extraction API route performs the real Document Intelligence call).
*
* Tenant-agnostic: mailbox address + Graph credentials + folder are all
* caller-supplied per invocation — nothing here is stored keyed only by a
* hardcoded mailbox. Nothing here assumes invoices, vendors, or Protosell;
* that logic (vendor-kb matching, dup-hash, DQ scoring, invoice
* classification) was intentionally left behind in
* v0-datahub/lib/invoice-inbox — it does not belong in a generic capture
* pipeline that also has to serve HR forms, contracts, receipts, etc.
*
* KNOWN GAP — ST0RE bytes storage: st0re-client.ts registers artifact
* METADATA only (a pointer to provider/bucket/key); it has no real
* bytes-upload endpoint yet. Until one exists, this service writes
* attachment bytes to LocalBlobStore (src/adapters/local-blob-store.ts), a
* clearly-marked interim local-disk store, then registers that location
* with ST0RE so the artifact/lineage metadata is still correct. Search this
* file for "INTERIM" to find the seam to replace.
*
* Singleton note: unlike source-service.ts / dataset-service.ts (which are
* instantiated fresh per API route file because each request is
* self-contained), DocumentCaptureService holds the only in-memory record
* of *which bytes belong to which document_id* — document-extraction-service
* must read the exact same instance to find them. So this module also
* exports a shared `documentCaptureService` singleton for both API routes
* to import, instead of each route constructing its own instance.
*/
import crypto from "crypto";
import { GraphMailboxAdapter, type GraphMailboxConfig, type GraphMessage, type GraphAttachment } from "../adapters/graph-mailbox-adapter";
import { LocalBlobStore } from "../adapters/local-blob-store";
import { createSt0reClient, type St0reClient } from "../clients/st0re-client";
import { createN0dClient, type N0dClient } from "../clients/n0d-client";
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import type { DocumentRecord, DocumentStatus } from "../types/document";
export interface CaptureMailboxContext {
tenant_id: string;
project_id: string;
actor_id: string;
correlation_id: string;
}
export interface CaptureMailboxInput {
mailbox: GraphMailboxConfig;
since_iso?: string;
top?: number;
max_pages?: number;
}
export interface CaptureMailboxResult {
scanned_messages: number;
captured_documents: DocumentRecord[];
errors: string[];
}
interface StoredBytesLocation {
bucket: string;
key: string;
}
export class DocumentCaptureService {
private documents: Map<string, DocumentRecord> = new Map();
private bytesByDocumentId: Map<string, StoredBytesLocation> = new Map();
private st0re: St0reClient;
private n0d: N0dClient;
private aud0: Aud0Client;
private blobStore: LocalBlobStore;
constructor(opts?: { st0reBaseUrl?: string; n0dBaseUrl?: string; aud0BaseUrl?: string; blobStore?: LocalBlobStore }) {
this.st0re = createSt0reClient(opts?.st0reBaseUrl);
this.n0d = createN0dClient(opts?.n0dBaseUrl);
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.blobStore = opts?.blobStore ?? new LocalBlobStore();
}
async captureFromMailbox(input: CaptureMailboxInput, ctx: CaptureMailboxContext): Promise<CaptureMailboxResult> {
if (!ctx.tenant_id || !ctx.project_id || !ctx.actor_id) {
throw new Error("captureFromMailbox: missing tenant_id/project_id/actor_id");
}
if (!GraphMailboxAdapter.isConfigured(input.mailbox)) {
throw new Error(
"captureFromMailbox: missing mailbox.tenantId/clientId/clientSecret/mailboxAddress",
);
}
const graph = new GraphMailboxAdapter(input.mailbox);
const result: CaptureMailboxResult = { scanned_messages: 0, captured_documents: [], errors: [] };
const messages = await graph.pollMessages({
sinceISO: input.since_iso,
top: input.top,
maxPages: input.max_pages,
});
for (const message of messages) {
result.scanned_messages++;
if (!message.hasAttachments) continue;
try {
const attachments = await graph.listAttachments(message.id);
for (const attachment of attachments) {
if (!attachment.isPdf) continue;
const document = await this.captureAttachment(input.mailbox, message, attachment, ctx);
result.captured_documents.push(document);
}
} catch (e: unknown) {
const msg = e instanceof Error ? e.message : String(e);
result.errors.push(`${message.id}: ${msg.slice(0, 200)}`);
}
}
return result;
}
private async captureAttachment(
mailbox: GraphMailboxConfig,
message: GraphMessage,
attachment: GraphAttachment,
ctx: CaptureMailboxContext,
): Promise<DocumentRecord> {
const documentId = `doc_${crypto.randomUUID()}`;
const now = new Date().toISOString();
const bucket = "0pt-documents";
const safeName = (attachment.name || "attachment.pdf").replace(/[^A-Za-z0-9._-]/g, "_");
const key = `${ctx.tenant_id}/${ctx.project_id}/${message.id}/${attachment.id}/${safeName}`;
// INTERIM — see file header. Real bytes persistence until ST0RE exposes an upload endpoint.
const written = await this.blobStore.write(bucket, key, attachment.contentBytes);
const checksum = crypto.createHash("sha256").update(attachment.contentBytes).digest("hex");
let record: DocumentRecord = {
document_id: documentId,
tenant_id: ctx.tenant_id,
project_id: ctx.project_id,
mailbox_address: mailbox.mailboxAddress,
graph_message_id: message.id,
graph_attachment_id: attachment.id,
filename: attachment.name ?? undefined,
content_type: attachment.contentType ?? undefined,
size_bytes: attachment.sizeBytes ?? attachment.contentBytes.length,
received_at: message.receivedAt ?? undefined,
sender_email: message.senderEmail ?? undefined,
sender_name: message.senderName ?? undefined,
subject: message.subject ?? undefined,
status: "captured",
correlation_id: ctx.correlation_id,
created_at: now,
};
this.documents.set(documentId, record);
this.bytesByDocumentId.set(documentId, { bucket: written.bucket, key: written.key });
// Register the artifact pointer with ST0RE (metadata only — see file header gap note).
const artifact = await this.st0re.createArtifact({
tenant_id: ctx.tenant_id,
project_id: ctx.project_id,
artifact_type: "document",
app_id: "0pt",
produced_by: "0pt.capture_document",
object_location: { provider: written.provider, bucket: written.bucket, key: written.key, region: "local" },
content_type: attachment.contentType || "application/pdf",
size_bytes: written.sizeBytes,
checksum_sha256: checksum,
pii_level: "high",
retention_policy: "document_capture_standard",
data_residency: "eu",
risk_level: "L3",
correlation_id: ctx.correlation_id,
});
// Enqueue extraction. A worker (out of scope here) or a caller hitting the
// /documents/[documentId]/extraction route performs the real DI call.
const job = await this.n0d.createJob({
tenant_id: ctx.tenant_id,
project_id: ctx.project_id,
job_type: "0pt.document_extraction_job",
capability_id: "0pt.extract_document",
requested_by_app_id: "0pt",
requested_by_actor_id: ctx.actor_id,
input_artifact_ids: [artifact.artifact_id],
output_artifact_type: "document_extraction",
requires_backbone: true,
requires_gpu: false,
required_node_roles: ["compute_general"],
resource_profile: "cpu_io_medium",
risk_level: "L3",
correlation_id: ctx.correlation_id,
});
record = {
...record,
st0re_artifact_id: artifact.artifact_id,
n0d_job_id: job.job_id,
status: "extraction_requested",
updated_at: new Date().toISOString(),
};
this.documents.set(documentId, record);
await this.aud0.ingestEvent({
event_id: `evt_0pt_document_captured_${documentId}`,
event_type: "0pt.document_captured",
tenant_id: ctx.tenant_id,
project_id: ctx.project_id,
actor_id: ctx.actor_id,
app_id: "0pt",
capability_id: "0pt.capture_document",
resource_type: "document",
resource_id: documentId,
risk_level: "L3",
payload: {
mailbox_address: mailbox.mailboxAddress,
graph_message_id: message.id,
filename: attachment.name,
size_bytes: written.sizeBytes,
st0re_artifact_id: artifact.artifact_id,
n0d_job_id: job.job_id,
},
created_at: new Date().toISOString(),
correlation_id: ctx.correlation_id,
});
await this.aud0.ingestLineageEvent({
lineage_event_id: `lineage_0pt_document_captured_${documentId}`,
tenant_id: ctx.tenant_id,
project_id: ctx.project_id,
app_id: "0pt",
capability_id: "0pt.capture_document",
source_type: "mailbox",
source_id: mailbox.mailboxAddress,
target_type: "document",
target_id: documentId,
lineage_type: "produced",
metadata: { graph_message_id: message.id, st0re_artifact_id: artifact.artifact_id },
created_at: new Date().toISOString(),
correlation_id: ctx.correlation_id,
});
return record;
}
async getDocument(documentId: string): Promise<DocumentRecord> {
const record = this.documents.get(documentId);
if (!record) throw new Error(`Document ${documentId} not found`);
return record;
}
async listDocuments(filter?: { tenant_id?: string; project_id?: string; status?: DocumentStatus }): Promise<DocumentRecord[]> {
let results = Array.from(this.documents.values());
if (filter?.tenant_id) results = results.filter((r) => r.tenant_id === filter.tenant_id);
if (filter?.project_id) results = results.filter((r) => r.project_id === filter.project_id);
if (filter?.status) results = results.filter((r) => r.status === filter.status);
return results;
}
/** Read back the raw bytes captured for a document. INTERIM local store — see file header. */
async getDocumentBytes(documentId: string): Promise<Buffer> {
const loc = this.bytesByDocumentId.get(documentId);
if (!loc) throw new Error(`Document ${documentId} has no stored bytes`);
return this.blobStore.read(loc.bucket, loc.key);
}
setStatus(documentId: string, status: DocumentStatus): void {
const record = this.documents.get(documentId);
if (!record) throw new Error(`Document ${documentId} not found`);
record.status = status;
record.updated_at = new Date().toISOString();
}
}
/** Shared singleton — see the "Singleton note" in the file header for why. */
export const documentCaptureService = new DocumentCaptureService();
+290
View File
@@ -0,0 +1,290 @@
/**
* src/services/document-extraction-service.ts
* ==============================================
* Evidence-based field extraction for a captured document.
*
* Calls DocumentIntelligenceAdapter and NEVER fabricates a value: a field
* key is only populated with a non-null value when Document Intelligence
* actually returned one, and it always carries DI's own confidence score
* (or null when DI reported none). Fields DI did not find are left blank —
* { value: null, confidence: <DI's confidence if any>, found: false } —
* never defaulted, guessed, or backfilled from anything else. This mirrors
* the evidence-based discipline in
* v0-datahub/lib/invoice-inbox/doc-intelligence.ts exactly (it never
* invented an amount/vendor/date that DI didn't return); it is a
* correctness property, not a style preference, so it is enforced twice —
* once inside the adapter's field normalizer, and again defensively here.
*
* modelId is caller-supplied ("prebuilt-invoice", "prebuilt-receipt",
* "prebuilt-layout", or a custom model id) — this service does not assume
* any particular document type. Invoice-specific reasoning (vendor
* matching, dup-hash, DQ scoring, invoice classification) was intentionally
* left in v0-datahub/lib/invoice-inbox and is NOT ported here.
*/
import crypto from "crypto";
import {
DocumentIntelligenceAdapter,
type DocumentIntelligenceConfig,
type ExtractedField as AdapterField,
} from "../adapters/document-intelligence-adapter";
import { LocalBlobStore } from "../adapters/local-blob-store";
import { createCl0udClient, type Cl0udClient } from "../clients/cl0ud-client";
import { createSt0reClient, type St0reClient } from "../clients/st0re-client";
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import { DocumentCaptureService, documentCaptureService } from "./document-capture-service";
import type { DocumentExtractionRecord, ExtractedField } from "../types/document-extraction";
export interface ExtractDocumentContext {
tenant_id: string;
project_id: string;
actor_id: string;
correlation_id: string;
/** e.g. "prebuilt-invoice" | "prebuilt-receipt" | "prebuilt-layout" | a custom model id. */
model_id: string;
}
/**
* Defensive re-application of the "never fabricate" rule at the service
* boundary. Even though the adapter's normalizeField() already enforces
* this, a field is only ever considered found here if it is BOTH marked
* found by the adapter AND carries a genuinely non-empty value — so a
* malformed/partial adapter response can never leak a fabricated value
* through this service.
*/
function toEvidenceFields(fields: Record<string, AdapterField>): Record<string, ExtractedField> {
const out: Record<string, ExtractedField> = {};
for (const [key, f] of Object.entries(fields)) {
const found = !!f?.found && f.value !== null && f.value !== undefined && f.value !== "";
out[key] = { value: found ? f.value : null, confidence: f?.confidence ?? null, found };
}
return out;
}
export class DocumentExtractionService {
private extractions: Map<string, DocumentExtractionRecord> = new Map();
private cl0ud: Cl0udClient;
private st0re: St0reClient;
private aud0: Aud0Client;
private blobStore: LocalBlobStore;
private di: DocumentIntelligenceAdapter | null;
constructor(
private captureService: DocumentCaptureService,
opts?: {
cl0udBaseUrl?: string;
st0reBaseUrl?: string;
aud0BaseUrl?: string;
documentIntelligence?: DocumentIntelligenceConfig;
blobStore?: LocalBlobStore;
},
) {
this.cl0ud = createCl0udClient(opts?.cl0udBaseUrl);
this.st0re = createSt0reClient(opts?.st0reBaseUrl);
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.blobStore = opts?.blobStore ?? new LocalBlobStore();
this.di =
opts?.documentIntelligence && DocumentIntelligenceAdapter.isConfigured(opts.documentIntelligence)
? new DocumentIntelligenceAdapter(opts.documentIntelligence)
: null;
}
async extractDocument(documentId: string, ctx: ExtractDocumentContext): Promise<DocumentExtractionRecord> {
if (!this.di) {
throw new Error(
"DocumentExtractionService: Document Intelligence is not configured (missing endpoint/apiKey) — refusing to fabricate a result",
);
}
if (!ctx.model_id) {
throw new Error("extractDocument: missing model_id");
}
if (!ctx.tenant_id || !ctx.project_id || !ctx.actor_id) {
throw new Error("extractDocument: missing tenant_id/project_id/actor_id");
}
const document = await this.captureService.getDocument(documentId);
// The caller's verified tenant alone is insufficient: the document being
// read must belong to the same tenant AND project before policy or bytes.
if (document.tenant_id !== ctx.tenant_id || document.project_id !== ctx.project_id) {
throw new Error(`Document ${documentId} not found`);
}
const policy = await this.cl0ud.evaluatePolicy({
tenant_id: ctx.tenant_id,
project_id: ctx.project_id,
actor_id: ctx.actor_id,
capability_id: "0pt.extract_document",
resource_type: "document",
resource_id: documentId,
risk_level: "L3",
correlation_id: ctx.correlation_id,
});
if (policy.decision === "deny") throw new Error(`Policy denied: ${policy.reason}`);
if (policy.decision === "approval_required") throw new Error(`Policy requires approval: ${policy.reason}`);
const extractionId = `dex_${crypto.randomUUID()}`;
const now = new Date().toISOString();
let record: DocumentExtractionRecord = {
extraction_id: extractionId,
document_id: documentId,
tenant_id: ctx.tenant_id,
project_id: ctx.project_id,
model_id: ctx.model_id,
status: "requested",
fields: {},
line_items: [],
fields_found_count: 0,
fields_total_count: 0,
average_confidence: null,
document_confidence: null,
correlation_id: ctx.correlation_id,
created_at: now,
};
this.extractions.set(extractionId, record);
try {
const bytes = await this.captureService.getDocumentBytes(documentId);
const diResult = await this.di.analyzeDocument(bytes, ctx.model_id);
const fields = toEvidenceFields(diResult.fields);
const lineItems = Object.values(diResult.arrayFields).flatMap((items) => items.map((item) => toEvidenceFields(item)));
const foundCount = Object.values(fields).filter((f) => f.found).length;
// Average only over FOUND fields — a not-found field's confidence (e.g. "31% sure this
// region isn't a date after all") describes DI's certainty about absence, not a value,
// and must not drag down the confidence of the values that were actually extracted.
const confidences = Object.values(fields)
.filter((f) => f.found)
.map((f) => f.confidence)
.filter((c): c is number => c != null);
const averageConfidence = confidences.length ? confidences.reduce((a, b) => a + b, 0) / confidences.length : null;
// Persist the extraction result as an artifact too — same interim-storage gap as
// document-capture-service.ts (see that file's header for the full explanation).
const payloadJson = Buffer.from(
JSON.stringify({ model_id: ctx.model_id, fields, line_items: lineItems, content: diResult.content }),
"utf8",
);
const bucket = "0pt-document-extractions";
const key = `${ctx.tenant_id}/${ctx.project_id}/${documentId}/${extractionId}.json`;
const written = await this.blobStore.write(bucket, key, payloadJson);
const artifact = await this.st0re.createArtifact({
tenant_id: ctx.tenant_id,
project_id: ctx.project_id,
artifact_type: "document_extraction",
app_id: "0pt",
produced_by: "0pt.extract_document",
object_location: { provider: written.provider, bucket: written.bucket, key: written.key, region: "local" },
content_type: "application/json",
size_bytes: written.sizeBytes,
checksum_sha256: crypto.createHash("sha256").update(payloadJson).digest("hex"),
pii_level: "high",
retention_policy: "document_extraction_standard",
data_residency: "eu",
risk_level: "L3",
correlation_id: ctx.correlation_id,
});
record = {
...record,
status: "completed",
fields,
line_items: lineItems,
fields_found_count: foundCount,
fields_total_count: Object.keys(fields).length,
average_confidence: averageConfidence,
document_confidence: diResult.documentConfidence,
st0re_artifact_id: artifact.artifact_id,
updated_at: new Date().toISOString(),
};
this.extractions.set(extractionId, record);
this.captureService.setStatus(documentId, "extracted");
await this.aud0.ingestEvent({
event_id: `evt_0pt_document_extracted_${extractionId}`,
event_type: "0pt.document_extracted",
tenant_id: ctx.tenant_id,
project_id: ctx.project_id,
actor_id: ctx.actor_id,
app_id: "0pt",
capability_id: "0pt.extract_document",
resource_type: "document_extraction",
resource_id: extractionId,
risk_level: "L3",
payload: {
document_id: documentId,
model_id: ctx.model_id,
fields_found_count: foundCount,
fields_total_count: Object.keys(fields).length,
average_confidence: averageConfidence,
st0re_artifact_id: artifact.artifact_id,
},
created_at: new Date().toISOString(),
correlation_id: ctx.correlation_id,
});
await this.aud0.ingestLineageEvent({
lineage_event_id: `lineage_0pt_document_extracted_${extractionId}`,
tenant_id: ctx.tenant_id,
project_id: ctx.project_id,
app_id: "0pt",
capability_id: "0pt.extract_document",
source_type: "document",
source_id: documentId,
target_type: "document_extraction",
target_id: extractionId,
lineage_type: "derived_from",
metadata: { model_id: ctx.model_id, st0re_artifact_id: artifact.artifact_id },
created_at: new Date().toISOString(),
correlation_id: ctx.correlation_id,
});
return record;
} catch (e: unknown) {
const message = e instanceof Error ? e.message : String(e);
record = { ...record, status: "failed", error: message, updated_at: new Date().toISOString() };
this.extractions.set(extractionId, record);
try {
this.captureService.setStatus(documentId, "extraction_failed");
} catch {
/* document may already be gone in edge-case tests — extraction failure record still stands */
}
throw e;
}
}
async getExtraction(extractionId: string): Promise<DocumentExtractionRecord> {
const record = this.extractions.get(extractionId);
if (!record) throw new Error(`Document extraction ${extractionId} not found`);
return record;
}
// P014: `tenantId` is optional so existing trusted in-process callers are
// unaffected, but the HTTP route now always supplies it -- closing a real
// gap: this used to be reachable by documentId alone with NO tenant
// check anywhere, so any authenticated caller could list another
// tenant's OCR/structuring extraction results (potentially containing
// extracted document content, e.g. invoice fields) by guessing or
// enumerating documentId. A mismatched tenantId reports identically to
// "not found" (no existence oracle), same convention as the rest of the
// surface.
async listExtractionsForDocument(documentId: string, tenantId?: string): Promise<DocumentExtractionRecord[]> {
if (tenantId) {
const document = await this.captureService.getDocument(documentId).catch(() => undefined);
if (!document || document.tenant_id !== tenantId) {
throw new Error(`Document ${documentId} not found`);
}
}
return Array.from(this.extractions.values()).filter((r) => r.document_id === documentId);
}
}
function buildDocumentIntelligenceConfigFromEnv(): DocumentIntelligenceConfig | undefined {
const endpoint = process.env.DOC_INTELLIGENCE_ENDPOINT;
const apiKey = process.env.DOC_INTELLIGENCE_KEY;
if (!endpoint || !apiKey) return undefined;
return { endpoint, apiKey, apiVersion: process.env.DOC_INTELLIGENCE_API_VERSION };
}
/** Shared singleton (see document-capture-service.ts "Singleton note") wired to the shared capture service. */
export const documentExtractionService = new DocumentExtractionService(documentCaptureService, {
documentIntelligence: buildDocumentIntelligenceConfigFromEnv(),
});
+10
View File
@@ -0,0 +1,10 @@
import { createN0dClient } from "../clients/n0d-client";
import { EntityResolutionService } from "../er/entity-resolution-service";
export const entityResolutionService = new EntityResolutionService();
let batchService: EntityResolutionService | undefined;
export function getBatchEntityResolutionService(): EntityResolutionService {
if (!batchService) batchService = new EntityResolutionService({ n0dClient: createN0dClient() });
return batchService;
}
+262
View File
@@ -0,0 +1,262 @@
/**
* src/services/external-writeback-service.ts
* =============================================
* `0pt.write_external_record` -- "POST a validated, already-approved
* structured record to a registered external-system adapter." Generic in its
* core: this service has no Odoo/invoice/Protosell-specific logic anywhere
* in it -- it resolves an `adapter_id` to a registered `WritebackAdapter`
* (../adapters/writeback-adapter.ts) and delegates the actual write to it.
* `../adapters/odoo-bills-adapter.ts`'s OdooBillsAdapter is ONE such
* adapter, registered by a caller the same way any other adapter would be --
* this service never imports it directly.
*
* WHY THIS EXISTS / WHERE IT SITS IN THE CHAIN: `act0/src/
* approval-writeback-templates.ts`'s generic `0pt_document_approval_then_
* external_write` Template chains a captured+extracted document
* (0pt.document_ocr's real terminal event, `0pt.document_extracted`) through
* an Approval Engine gate, THEN calls this capability. By the time
* `writeExternalRecord` runs, a human has already approved the write -- this
* service still re-checks CL0UD policy (per docs/PRODUCT_PLAN.md section
* 4.1's hard rule that CL0UD is always re-consulted, the same rule
* document-extraction-service.ts already follows -- confirmed by grepping
* this repo's src/services/ for evaluatePolicy()/Cl0udClient usage; the
* only other real service that does the same is iceberg-write-service.ts.
* An earlier version of this comment also cited `quality-check-service.ts`
* here; that file does not exist anywhere in this repo),
* but does not itself re-verify the upstream Approval record (ACT0 owns
* approvals; 0pt's own capability boundary is "did CL0UD say yes for THIS
* call", same separation of concerns every other 0pt service in this repo
* already draws).
*
* IDEMPOTENCY (per this task's own instruction not to regress the proven
* pattern ported from v0-datahub's `lib/odoo/invoice-bridge.ts`
* `processInboxToOdoo`): that original code CLAIMS a row (an atomic
* Postgres `UPDATE ... WHERE odoo_move_id IS NULL AND odoo_state != 'posting'`)
* BEFORE ever calling the external system, so two concurrent triggers (a
* manual approve racing a cron, or a retried ACT0 step racing itself) cannot
* both post the same record. This service ports that SAME "claim before you
* call out" shape forward as a generic, adapter-agnostic in-memory claim,
* keyed on `(adapter_id, idempotency_key)`:
* - the claim check-and-set happens SYNCHRONOUSLY, before this function's
* first `await` -- JavaScript's single-threaded run-to-first-await
* semantics make this exactly as race-free, within one process, as the
* original's atomic SQL UPDATE was across concurrent DB clients (see
* `writeExternalRecord`'s own inline comment for exactly where the line
* is drawn, and tests/p0/external-writeback-service.test.ts's own
* concurrency test, which calls `writeExternalRecord` twice with the
* same idempotency key via `Promise.all` and asserts the underlying
* adapter's `write` is invoked exactly once).
* - HONEST SCOPE: this in-memory claim is process-local -- it does not
* survive a restart and does not protect a multi-instance deployment,
* unlike the original's real Postgres row lock. An adapter with its own
* persistent storage (OdooBillsAdapter) additionally refuses to
* re-post a record whose payload already carries an `existing_external_id`
* -- see WritebackAdapter.write's own doc comment for why this
* second layer matters and odoo-bills-adapter.ts for exactly how it is
* ported. A production deployment backing this service with real
* persistent storage (same "pure core, thin adapter" doctrine every
* other 0pt service in this repo already documents) should replace this
* in-memory Map with a real claim row, at which point the SAME
* synchronous-claim-before-any-await shape still applies, now backed by
* a real transaction instead of a JS Map.
*/
import crypto from "crypto";
import { createCl0udClient, type Cl0udClient } from "../clients/cl0ud-client";
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import type { WritebackAdapter } from "../adapters/writeback-adapter";
import type { ExternalWriteRecord, ExternalWriteRequest } from "../types/external-writeback";
export interface ExternalWritebackServiceOptions {
cl0udBaseUrl?: string;
aud0BaseUrl?: string;
adapters?: readonly WritebackAdapter[];
}
export class ExternalWritebackService {
private writes: Map<string, ExternalWriteRecord> = new Map();
/** `${adapter_id}::${idempotency_key}` -> write_id. See this module's own doc comment's
* "IDEMPOTENCY" section for exactly what this does and does not guarantee. */
private claimedByIdempotencyKey: Map<string, string> = new Map();
private adapters: Map<string, WritebackAdapter> = new Map();
private cl0ud: Cl0udClient;
private aud0: Aud0Client;
constructor(opts?: ExternalWritebackServiceOptions) {
this.cl0ud = createCl0udClient(opts?.cl0udBaseUrl);
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
for (const adapter of opts?.adapters ?? []) {
this.registerAdapter(adapter);
}
}
registerAdapter(adapter: WritebackAdapter): void {
this.adapters.set(adapter.adapterId, adapter);
}
getAdapter(adapterId: string): WritebackAdapter {
const adapter = this.adapters.get(adapterId);
if (!adapter) throw new Error(`ExternalWritebackService: no adapter registered for '${adapterId}'`);
return adapter;
}
async writeExternalRecord(request: ExternalWriteRequest): Promise<ExternalWriteRecord> {
const adapter = this.adapters.get(request.adapter_id);
if (!adapter) {
throw new Error(`ExternalWritebackService: no adapter registered for '${request.adapter_id}'`);
}
if (!adapter.isConfigured()) {
throw new Error(
`ExternalWritebackService: adapter '${request.adapter_id}' is not configured — refusing to fabricate a write`,
);
}
// --- SYNCHRONOUS claim section: no `await` above this line, and none until the claim is
// set below. This is what makes the claim race-free under this module's own doc comment's
// "IDEMPOTENCY" doctrine -- see that comment for the full explanation and its own scope
// caveat (process-local only).
const claimKey = `${request.adapter_id}::${request.idempotency_key}`;
const existingWriteId = this.claimedByIdempotencyKey.get(claimKey);
if (existingWriteId) {
const existing = this.writes.get(existingWriteId);
if (existing && existing.status === "written") {
return existing; // already posted -- idempotent no-op, matches invoice-bridge.ts's own
// "Redan bokförd i Odoo... hoppar över" early return.
}
if (existing && existing.status === "claimed") {
// A write for this exact idempotency key is already in flight (or, in this in-memory
// reference implementation, was left 'claimed' by a caller that never awaited the
// result) -- do not race a second POST to the external system. Matches
// invoice-bridge.ts's own "Redan under postning eller bokförd — hoppar över" claim-
// conflict response.
return { ...existing, status: "skipped_duplicate" };
}
// existing.status === 'failed' (or the record is missing for any other reason): fall
// through and re-claim -- a failed write is safe and expected to retry, matching
// invoice-bridge.ts's own "Release the 'posting' claim... can be retried later" behavior.
}
const writeId = `ewr_${crypto.randomUUID()}`;
const now = new Date().toISOString();
let record: ExternalWriteRecord = {
write_id: writeId,
adapter_id: request.adapter_id,
idempotency_key: request.idempotency_key,
record_type: request.record_type,
status: "claimed",
tenant_id: request.tenant_id,
project_id: request.project_id,
actor_id: request.actor_id,
correlation_id: request.correlation_id,
created_at: now,
};
this.writes.set(writeId, record);
this.claimedByIdempotencyKey.set(claimKey, writeId);
// --- End synchronous claim section. Every branch below is async and safe to interleave. ---
try {
const policy = await this.cl0ud.evaluatePolicy({
tenant_id: request.tenant_id,
project_id: request.project_id,
actor_id: request.actor_id,
capability_id: "0pt.write_external_record",
resource_type: "external_write",
resource_id: writeId,
risk_level: "L4",
correlation_id: request.correlation_id,
});
if (policy.decision === "deny") {
throw new Error(`Policy denied: ${policy.reason}`);
}
if (policy.decision === "approval_required") {
// Per this service's own module doc comment: the ACT0 Approval Engine gate that
// precedes this capability in the approval-writeback-templates.ts chain is a SEPARATE
// mechanism from CL0UD's own policy decision here -- if CL0UD independently still says
// approval_required for this specific call, this service does not silently bypass it
// (same "never lie about what happened" doctrine every other 0pt service in this repo
// follows for a denied/approval_required decision).
throw new Error(`Policy requires approval: ${policy.reason}`);
}
const result = await adapter.write(
{ recordType: request.record_type, payload: request.payload, idempotencyKey: request.idempotency_key },
{
tenant_id: request.tenant_id,
project_id: request.project_id,
actor_id: request.actor_id,
correlation_id: request.correlation_id,
},
);
if (!result.ok) {
throw new Error(result.error ?? "adapter_write_failed");
}
record = { ...record, status: "written", external_id: result.externalId, updated_at: new Date().toISOString() };
this.writes.set(writeId, record);
await this.aud0.ingestEvent({
event_id: `evt_0pt_external_record_written_${writeId}`,
event_type: "0pt.external_record_written",
tenant_id: request.tenant_id,
project_id: request.project_id,
actor_id: request.actor_id,
app_id: "0pt",
capability_id: "0pt.write_external_record",
resource_type: "external_write",
resource_id: writeId,
risk_level: "L4",
payload: {
adapter_id: request.adapter_id,
record_type: request.record_type,
idempotency_key: request.idempotency_key,
external_id: result.externalId,
},
created_at: new Date().toISOString(),
correlation_id: request.correlation_id,
});
await this.aud0.ingestLineageEvent({
lineage_event_id: `lineage_0pt_external_write_${writeId}`,
tenant_id: request.tenant_id,
project_id: request.project_id,
app_id: "0pt",
capability_id: "0pt.write_external_record",
// `idempotency_key` is, by convention (not enforced by this generic service), the
// upstream record's stable id -- e.g. a document_extraction_id when this write was
// triggered by act0's document_approval_then_external_write template.
source_type: "document_extraction",
source_id: request.idempotency_key,
target_type: "external_record",
target_id: result.externalId ?? writeId,
lineage_type: "produced",
metadata: { adapter_id: request.adapter_id, record_type: request.record_type },
created_at: new Date().toISOString(),
correlation_id: request.correlation_id,
});
return record;
} catch (e: unknown) {
const message = e instanceof Error ? e.message : String(e);
record = { ...record, status: "failed", error: message, updated_at: new Date().toISOString() };
this.writes.set(writeId, record);
throw e;
}
}
async getWrite(writeId: string): Promise<ExternalWriteRecord> {
const record = this.writes.get(writeId);
if (!record) throw new Error(`External write ${writeId} not found`);
return record;
}
async findByIdempotencyKey(adapterId: string, idempotencyKey: string): Promise<ExternalWriteRecord | undefined> {
const writeId = this.claimedByIdempotencyKey.get(`${adapterId}::${idempotencyKey}`);
return writeId ? this.writes.get(writeId) : undefined;
}
}
/** Shared singleton, same "cross-route in-memory state" convention document-capture-service.ts's
* own header documents (this service's writes/claims must be visible across whatever API routes
* call writeExternalRecord vs getWrite) -- no adapters pre-registered; a caller (an API route,
* a test, or a future 0pt/api/documents/writeback route) registers whichever adapters it needs
* via registerAdapter before calling writeExternalRecord. */
export const externalWritebackService = new ExternalWritebackService();
+342
View File
@@ -0,0 +1,342 @@
import type { DatasetIcebergMapping } from "../types/dataset-iceberg-mapping";
import { createCl0udClient, type Cl0udClient } from "../clients/cl0ud-client";
import { createN0dClient, type N0dClient } from "../clients/n0d-client";
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import {
createSt0reLakehouseClient,
type St0reLakehouseClient,
type IcebergTableFormat,
type TableSnapshotType,
} from "../clients/st0re-lakehouse-client";
import { DatasetService } from "./dataset-service";
import { DatasetIcebergMappingService } from "./dataset-iceberg-mapping-service";
/**
* IcebergWriteService -- 0pt's write-path entrypoint for Iceberg tables.
*
* This is the concrete implementation of docs/ICEBERG_WRITE_BOUNDARY.md /
* docs/11-lakehouse-write-plan.md items 1, 3, 6, 7, 8: 0pt is the canonical
* writer of Iceberg tables, and it writes THROUGH ST0RE's table registry
* (`St0reLakehouseClient`) rather than persisting any lakehouse table,
* catalog, or snapshot state itself. This service holds NO table/snapshot
* state of its own -- only `DatasetIcebergMappingService`'s lightweight
* dataset-to-table-id pointer, which is a reference, not a copy.
*
* Two entry points:
* - `mapDatasetToIcebergTable` -- registers a NEW Iceberg table in ST0RE for
* a dataset that does not have one yet, then records the mapping. This is
* the "first test dataset written to Iceberg through ST0RE's table
* registry" proof path (see tests/p0/iceberg-write-service.test.ts).
* - `requestIcebergWrite` -- registers a new snapshot (an actual write) for
* an already-mapped dataset, optionally routed through an N0D job when the
* write is heavy (item 7: 0pt REQUESTS N0D jobs for heavy writes/
* compactions, it does not execute them itself, and it never owns the
* result of that job beyond the ST0RE-registered snapshot ref).
*
* Byte-level writes (actually producing Parquet/Iceberg data files) are OUT
* OF SCOPE for this pass, same honest boundary ST0RE itself documents for
* its own snapshot routes ("ST0RE doesn't itself write the Iceberg
* metadata.json... this registry tracks the pointer/version history for
* whatever the caller's own catalog write produced" --
* sax3l/st0re src/api/routes/lakehouse.ts). What this service proves is the
* OWNERSHIP path -- that 0pt requests the write and registers its result
* through ST0RE, never anywhere else.
*/
export interface MapDatasetToIcebergTableInput {
tenant_id: string;
project_id: string;
dataset_id: string;
dataset_iceberg_mapping_id: string;
namespace: string;
table_name: string;
storage_uri: string;
catalog_ref: string;
table_format?: IcebergTableFormat;
actor_id: string;
correlation_id: string;
}
export interface MapDatasetToIcebergTableResult {
mapping: DatasetIcebergMapping;
iceberg_table_id: string;
namespace_id: string;
policy_decision_id: string;
}
export interface RequestIcebergWriteInput {
dataset_iceberg_mapping_id: string;
/**
* P014: optional so existing trusted in-process callers are unaffected,
* but the HTTP route (app/api/0pt/lakehouse/mappings/[mappingId]/writes)
* always supplies it now. When present, it must match the mapping's own
* tenant_id or the write is refused as not-found -- see getMapping's
* doc comment for the gap this closes (a foreign mapping_id used to let
* any caller register a snapshot against another tenant's table).
*/
tenant_id?: string;
actor_id: string;
correlation_id: string;
snapshot_type?: TableSnapshotType;
schema_version: string;
manifest_uri: string;
record_count?: number;
size_bytes?: number;
/** True for large backfills/table maintenance -- routes through an N0D job request rather than 0pt executing in-process (docs/11-lakehouse-write-plan.md item 7). */
heavy?: boolean;
}
export interface RequestIcebergWriteResult {
snapshot_id: string;
iceberg_table_id: string;
policy_decision_id: string;
n0d_job_id?: string;
}
export class IcebergWriteService {
private cl0ud: Cl0udClient;
private n0d: N0dClient;
private st0reLakehouse: St0reLakehouseClient;
private aud0: Aud0Client;
private datasetService: DatasetService;
private mappingService: DatasetIcebergMappingService;
constructor(opts?: {
cl0udBaseUrl?: string;
n0dBaseUrl?: string;
st0reBaseUrl?: string;
aud0BaseUrl?: string;
datasetService?: DatasetService;
mappingService?: DatasetIcebergMappingService;
}) {
this.cl0ud = createCl0udClient(opts?.cl0udBaseUrl);
this.n0d = createN0dClient(opts?.n0dBaseUrl);
this.st0reLakehouse = createSt0reLakehouseClient(opts?.st0reBaseUrl);
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.datasetService = opts?.datasetService ?? new DatasetService(opts?.aud0BaseUrl);
this.mappingService = opts?.mappingService ?? new DatasetIcebergMappingService({ aud0BaseUrl: opts?.aud0BaseUrl, datasetService: this.datasetService });
}
async mapDatasetToIcebergTable(input: MapDatasetToIcebergTableInput): Promise<MapDatasetToIcebergTableResult> {
const dataset = await this.datasetService.getDataset(input.dataset_id); // throws if missing
// CL0UD policy boundary -- same L3 gate ST0RE's own table-registration
// route applies (sax3l/st0re src/api/routes/lakehouse.ts,
// 'st0re.lakehouse.write_table'), checked here too since 0pt is the
// caller initiating the write, not just relying on ST0RE to gate it.
const policy = await this.cl0ud.evaluatePolicy({
tenant_id: input.tenant_id,
project_id: input.project_id,
actor_id: input.actor_id,
capability_id: "0pt.write_iceberg_table",
resource_type: "dataset",
resource_id: input.dataset_id,
risk_level: "L3",
correlation_id: input.correlation_id,
});
if (policy.decision === "deny") throw new Error(`Policy denied: ${policy.reason}`);
if (policy.decision === "approval_required") throw new Error(`Policy requires approval: ${policy.reason}`);
const ctx = { tenantId: input.tenant_id, actorId: input.actor_id };
// Register (or reuse) the namespace THROUGH ST0RE's namespace registry
// first, so the mapping can carry a real `namespace_id` (ST0RE's
// LakehouseNamespace.namespaceId) rather than only the bare namespace
// name string. See findOrRegisterNamespace's doc comment for why this is
// a lookup-or-create, not a bare create.
const namespace = await this.findOrRegisterNamespace(input.tenant_id, input.project_id, input.namespace, ctx);
// Register the table THROUGH ST0RE's table registry -- 0pt does not
// create or track lakehouse table state on its own. NOTE: `datasetId` is
// deliberately NOT sent -- it is a foreign key into ST0RE's OWN
// dataset/artifact table (live-verified: sending 0pt's `dataset_id` here
// 400s with a Postgres FK violation, since 0pt datasets are never rows in
// ST0RE's dataset table). 0pt's dataset<->table linkage is carried
// entirely by the DatasetIcebergMapping row created below.
const table = await this.st0reLakehouse.registerTable(
{
tenantId: input.tenant_id,
projectId: input.project_id,
namespace: input.namespace,
tableName: input.table_name,
tableFormat: input.table_format ?? "iceberg", // ADR-0030: Iceberg is canonical; Delta/Hudi accepted only for bridge/import scenarios, never chosen by default here.
storageUri: input.storage_uri,
catalogRef: input.catalog_ref,
ownerProduct: "0pt",
createdBy: input.actor_id,
},
ctx
);
const mapping = await this.mappingService.createMapping({
dataset_iceberg_mapping_id: input.dataset_iceberg_mapping_id,
tenant_id: input.tenant_id,
project_id: input.project_id,
dataset_id: input.dataset_id,
iceberg_table_id: table.tableId,
namespace_id: namespace.namespaceId,
namespace: input.namespace,
table_name: input.table_name,
mapping_status: "active",
mapped_by: input.actor_id,
correlation_id: input.correlation_id,
});
return { mapping, iceberg_table_id: table.tableId, namespace_id: namespace.namespaceId, policy_decision_id: policy.decision_id };
}
async requestIcebergWrite(input: RequestIcebergWriteInput): Promise<RequestIcebergWriteResult> {
const mapping = await this.mappingService.getMapping(input.dataset_iceberg_mapping_id, input.tenant_id);
if (mapping.mapping_status !== "active") {
throw new Error(`Dataset Iceberg mapping ${input.dataset_iceberg_mapping_id} is not active (status=${mapping.mapping_status})`);
}
const policy = await this.cl0ud.evaluatePolicy({
tenant_id: mapping.tenant_id,
project_id: mapping.project_id,
actor_id: input.actor_id,
capability_id: "0pt.write_iceberg_table",
resource_type: "dataset",
resource_id: mapping.dataset_id,
risk_level: "L3",
correlation_id: input.correlation_id,
});
if (policy.decision === "deny") throw new Error(`Policy denied: ${policy.reason}`);
if (policy.decision === "approval_required") throw new Error(`Policy requires approval: ${policy.reason}`);
// 0pt.iceberg_write_requested -- fired when 0pt requests the write,
// before the write lands (docs/11-lakehouse-write-plan.md event list).
await this.aud0.ingestEvent({
event_id: `evt_0pt_iceberg_write_requested_${input.dataset_iceberg_mapping_id}_${Date.now()}`,
event_type: "0pt.iceberg_write_requested",
tenant_id: mapping.tenant_id,
project_id: mapping.project_id,
actor_id: input.actor_id,
app_id: "0pt",
capability_id: "0pt.write_iceberg_table",
resource_type: "iceberg_table",
resource_id: mapping.iceberg_table_id,
risk_level: "L3",
payload: { dataset_id: mapping.dataset_id, dataset_iceberg_mapping_id: mapping.dataset_iceberg_mapping_id, heavy: input.heavy ?? false },
created_at: new Date().toISOString(),
correlation_id: input.correlation_id,
});
// Heavy writes/compactions are REQUESTED from N0D, never executed
// in-process by 0pt (docs/11-lakehouse-write-plan.md item 7). N0D
// executes; it does not own the schema or the decision of what to
// write -- the snapshot registration below still goes through ST0RE
// regardless of who performed the write.
let n0dJobId: string | undefined;
if (input.heavy) {
const job = await this.n0d.createJob({
tenant_id: mapping.tenant_id,
project_id: mapping.project_id,
job_type: "0pt.iceberg_write_job",
capability_id: "0pt.write_iceberg_table",
requested_by_app_id: "0pt",
requested_by_actor_id: input.actor_id,
input_dataset_ids: [mapping.dataset_id],
output_artifact_type: "iceberg_table_snapshot",
requires_backbone: true,
requires_gpu: false,
required_node_roles: ["compute_general"],
resource_profile: "cpu_io_large",
risk_level: "L3",
correlation_id: input.correlation_id,
});
n0dJobId = job.job_id;
}
// Register the snapshot THROUGH ST0RE's table registry -- the one and
// only place 0pt persists that a write landed.
const snapshot = await this.st0reLakehouse.createSnapshot(
mapping.iceberg_table_id,
{
snapshotType: input.snapshot_type ?? "append",
schemaVersion: input.schema_version,
manifestUri: input.manifest_uri,
createdByJobId: n0dJobId,
createdByCapability: "0pt.write_iceberg_table",
recordCount: input.record_count,
sizeBytes: input.size_bytes,
},
{ tenantId: mapping.tenant_id, actorId: input.actor_id }
);
// 0pt.iceberg_snapshot_created -- fired once the write lands and ST0RE
// has registered a new snapshot, carrying table ref + snapshot id + row
// count + dataset id for AUD0 to build source -> dataset -> snapshot
// lineage.
await this.aud0.ingestEvent({
event_id: `evt_0pt_iceberg_snapshot_created_${snapshot.snapshotId}`,
event_type: "0pt.iceberg_snapshot_created",
tenant_id: mapping.tenant_id,
project_id: mapping.project_id,
actor_id: input.actor_id,
app_id: "0pt",
capability_id: "0pt.write_iceberg_table",
resource_type: "iceberg_snapshot",
resource_id: snapshot.snapshotId,
risk_level: "L2",
payload: {
dataset_id: mapping.dataset_id,
iceberg_table_id: mapping.iceberg_table_id,
snapshot_id: snapshot.snapshotId,
record_count: input.record_count,
n0d_job_id: n0dJobId,
},
created_at: new Date().toISOString(),
correlation_id: input.correlation_id,
});
await this.aud0.ingestLineageEvent({
lineage_event_id: `lineage_0pt_iceberg_snapshot_${snapshot.snapshotId}`,
tenant_id: mapping.tenant_id,
project_id: mapping.project_id,
app_id: "0pt",
capability_id: "0pt.write_iceberg_table",
source_type: "dataset",
source_id: mapping.dataset_id,
target_type: "iceberg_snapshot",
target_id: snapshot.snapshotId,
lineage_type: "produced",
metadata: { iceberg_table_id: mapping.iceberg_table_id, n0d_job_id: n0dJobId },
created_at: new Date().toISOString(),
correlation_id: input.correlation_id,
});
return {
snapshot_id: snapshot.snapshotId,
iceberg_table_id: mapping.iceberg_table_id,
policy_decision_id: policy.decision_id,
n0d_job_id: n0dJobId,
};
}
/**
* Looks up an existing ST0RE lakehouse namespace by (tenantId, projectId,
* name) before creating one. ST0RE has no upsert route and enforces a real
* unique constraint on that triple (live-verified 2026-07-13: a second
* `registerNamespace` call for the same triple 400s with
* `duplicate key value violates unique constraint
* "st0re_lakehouse_namespaces_tenant_id_project_id_name_key"`), so a bare
* "always create" call is not idempotent and would break every
* `mapDatasetToIcebergTable` call after the first for a given namespace.
*
* ST0RE's list route does not reliably scope or filter results (see the
* `listNamespaces` doc comment in st0re-lakehouse-client.ts, live-verified
* finding #4) so the match is done here, client-side, against the full
* returned list.
*/
private async findOrRegisterNamespace(
tenantId: string,
projectId: string,
name: string,
ctx: { tenantId: string; actorId: string }
) {
const existing = await this.st0reLakehouse.listNamespaces(ctx);
const match = existing.find((n) => n.tenantId === tenantId && n.projectId === projectId && n.name === name);
if (match) return match;
return this.st0reLakehouse.registerNamespace({ tenantId, projectId, name, ownerProduct: "0pt" }, ctx);
}
}
@@ -0,0 +1,19 @@
import { DatasetService } from "./dataset-service";
import { DatasetIcebergMappingService } from "./dataset-iceberg-mapping-service";
import { IcebergWriteService } from "./iceberg-write-service";
/**
* Shared singleton instances for the lakehouse-mapping route group
* (app/api/0pt/lakehouse/mappings/*). Previously each route file in that
* group instantiated its own DatasetService/DatasetIcebergMappingService/
* IcebergWriteService, so a mapping created via POST /mappings was invisible
* to GET /mappings/{id} and to POST /mappings/{id}/writes within the same
* server process (each held a distinct in-memory Map). Constructing all
* three here once, wired together, and importing them everywhere fixes that
* -- matching the in-memory-per-module convention this repo already uses
* elsewhere (see document-capture-service.ts's own singleton note), just
* applied consistently across this route group instead of per-file.
*/
export const datasetService = new DatasetService();
export const mappingService = new DatasetIcebergMappingService({ datasetService });
export const writeService = new IcebergWriteService({ datasetService, mappingService });
+371
View File
@@ -0,0 +1,371 @@
/**
* src/services/pii-gate-service.ts
* ==================================
* 0pt.pii_* — the GDPR guardrail (fail-closed). Enforced by default wherever
* a SourceQueryService is constructed; consulted by workers before they
* persist anything for a personal-data source.
*
* Owns 0pt.pii_policy.schema.v1 and the in-memory identity registry that
* backs the subject rights surfaces (access/erasure, GDPR art. 15/17) and
* the retention worker (GDPR art. 5(1)(e)). Every decision is emitted to
* AUD0 as risk L3 events so CL0UD gets an audit trail without polling 0pt.
*
* Directory domains known to publish personuppgifter (ratsit.se, …) are
* pre-classified as personal: until a PiiPolicy with a recorded lawful
* basis exists for them, the gate REJECTS storage/query of those sources.
*/
import { createHash } from "node:crypto";
import type {
PiiPolicy,
CreatePiiPolicyInput,
PiiIdentityKey,
PiiErasureRequest,
PiiAccessStatement,
ProcessingPurpose,
} from "../types/pii";
import type { Source } from "../types/source";
import { createAud0Client, type Aud0Client, type Aud0Event } from "../clients/aud0-client";
/** Domain publishes personuppgifter (directory/register listings). */
const KNOWN_DIRECTORY_DOMAINS = [
"ratsit.se",
"merinfo.se",
"hitta.se",
"mrkoll.se",
"eniro.se",
"birthday.se",
"upplysning.se",
];
const DECISION_TEMPLATE = "PII source %s: %s";
export class PiiPolicyExistsError extends Error {}
export class PiiLegalBasisError extends Error {}
export class PiiMinimisationError extends Error {}
export class PiiRetentionError extends Error {}
export class PiiRobotsRateError extends Error {}
export class PiiEntityLinkageError extends Error {}
export class PiiGateService {
private policies: Map<string, PiiPolicy> = new Map();
private registry: PiiIdentityKey[] = [];
private aud0: Aud0Client;
constructor(opts?: { aud0BaseUrl?: string }) {
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
}
/** True if the source either self-flags or is a known directory domain. */
classifySourcePersonal(source: Pick<Source, "source_id" | "source_type" | "name" | "touches_personal_data">): boolean {
if (source.touches_personal_data === true) return true;
if (source.source_type !== "web" && source.source_type !== "api") return false;
const host = (source.name || "").toLowerCase().replace(/^https?:\/\//, "").replace(/\/.*$/, "");
return KNOWN_DIRECTORY_DOMAINS.some((d) => host === d || host.endsWith("." + d));
}
/* ------------------------------------------------------------------ *
* Policies
* ------------------------------------------------------------------ */
async createPiiPolicy(input: CreatePiiPolicyInput): Promise<PiiPolicy> {
if (this.policies.has(input.source_id)) {
throw new Error(`PiiPolicy already exists for source ${input.source_id}`);
}
// Hard validation — a policy is the lawful-basis gate, so it cannot be
// written with a half-legal posture.
if (!input.basis_ref.trim()) throw new Error("basis_ref is required (LIA id, consent id, contract clause or legal provision)");
if (!Number.isFinite(input.retention_days) || input.retention_days < 0) {
throw new Error("retention_days must be a non-negative integer (0 = fetch-and-drop)");
}
if (!Array.isArray(input.field_allowlist) || input.field_allowlist.length === 0) {
throw new Error("field_allowlist must contain at least one field (data minimisation)");
}
// Recorded decision (Simon, 2026-08-31): legitimate interest is accepted for
// BUSINESS CONTACTS — personal data only in correspondence with a company/org
// entity. Enforce that posture so "B2B-kontakt, företagskopplad" cannot be
// bypassed by a lazily-written policy.
const purpose: ProcessingPurpose = input.purpose ?? "business_contact";
const entityLinkedOnly = input.entity_linked_only ?? input.legal_basis === "legitimate_interest";
if (input.legal_basis === "legitimate_interest") {
if (purpose !== "business_contact") {
throw new Error("legitimate_interest is only accepted with purpose=business_contact (recorded decision 2026-08-31)");
}
if (!entityLinkedOnly) {
throw new Error("legitimate_interest requires entity_linked_only=true — personal data must be bound to a company/org entity");
}
}
const now = new Date().toISOString();
const policy: PiiPolicy = {
source_id: input.source_id,
tenant_id: input.tenant_id,
project_id: input.project_id,
touches_personal_data: true,
legal_basis: input.legal_basis,
purpose,
entity_linked_only: entityLinkedOnly,
basis_ref: input.basis_ref,
retention_days: input.retention_days,
field_allowlist: [...input.field_allowlist],
pseudonymize_keys: input.pseudonymize_keys ?? true,
robots_and_rate_confirmed: input.robots_and_rate_confirmed ?? false,
created_at: now,
updated_at: null,
created_by: input.created_by ?? null,
};
this.policies.set(input.source_id, policy);
await this.safeEmit({
event_id: `evt_0pt_pii_policy_created_${input.source_id}`,
event_type: "0pt.pii_policy_created",
capability_id: "0pt.pii_policy_created",
tenant_id: input.tenant_id,
project_id: input.project_id,
actor_id: input.created_by ?? "system",
resource_type: "source_pii_policy",
resource_id: input.source_id,
risk_level: "L3",
payload: {
legal_basis: input.legal_basis,
purpose,
entity_linked_only: entityLinkedOnly,
retention_days: input.retention_days,
field_count: input.field_allowlist.length,
},
});
return policy;
}
async getPiiPolicy(sourceId: string): Promise<PiiPolicy | null> {
return this.policies.get(sourceId) ?? null;
}
async listPiiPolicies(tenantId?: string): Promise<PiiPolicy[]> {
const all = Array.from(this.policies.values());
return tenantId ? all.filter((p) => p.tenant_id === tenantId) : all;
}
/* ------------------------------------------------------------------ *
* Enforcement (fail-closed)
* ------------------------------------------------------------------ */
/**
* Gate for a store/query of `fields` against `source`. Throws unless the
* source is non-personal or a lawful, minimised, retention-bounded,
* robots/rate-confirmed policy is on record.
*/
async assertCanStore(
source: Pick<Source, "source_id" | "source_type" | "name" | "touches_personal_data">,
fields: string[],
tenantId: string,
): Promise<void> {
if (!this.classifySourcePersonal(source)) return;
const policy = this.policies.get(source.source_id);
if (!policy) {
throw new PiiLegalBasisError(
DECISION_TEMPLATE.replace("%s", source.source_id).replace(
"%s",
`is a personal-data source with no PiiPolicy on record — a lawful basis (${KNOWN_DIRECTORY_DOMAINS.includes(source.name || "") ? "including robots.txt + rate limit confirmation" : "art. 6 GDPR"}) must be registered before any processing`,
),
);
}
if (policy.retention_days === 0 && fields.length > 0) {
throw new PiiRetentionError(
DECISION_TEMPLATE.replace("%s", source.source_id).replace(
"%s",
"policy forbids retention (retention_days = 0); no fields may be stored",
),
);
}
const disallowed = fields.filter((f) => !policy.field_allowlist.includes(f));
if (disallowed.length > 0) {
throw new PiiMinimisationError(
DECISION_TEMPLATE.replace("%s", source.source_id).replace(
"%s",
`field(s) outside the allowlist: ${disallowed.join(", ")} (GDPR art. 5(1)(c) minimisation)`,
),
);
}
if ((source.source_type === "web" || source.source_type === "api") && !policy.robots_and_rate_confirmed) {
throw new PiiRobotsRateError(
DECISION_TEMPLATE.replace("%s", source.source_id).replace(
"%s",
"robots.txt / rate-limit not confirmed — required for web & api personal-data sources; re-check the source terms (do not hardcode)",
),
);
}
}
/* ------------------------------------------------------------------ *
* Subject rights: access (art. 15) & erasure (art. 17), retention (art. 5)
* ------------------------------------------------------------------ */
/**
* Register that one value was stored. Only hashes are persisted; the
* plaintext identity never reaches the registry.
*/
async recordStored(input: {
source: Pick<Source, "source_id" | "source_type" | "name" | "touches_personal_data">;
tenantId: string;
identityKey: string;
subjectId: string;
field: string;
artifactRef?: string;
/** Company/org entity the value is bound to (orgnr/rgs/LEI). REQUIRED when the policy is entity_linked_only. */
entityId?: string;
}): Promise<PiiIdentityKey> {
await this.assertCanStore(input.source, [input.field], input.tenantId);
const policy = this.policies.get(input.source.source_id)!;
if (policy.entity_linked_only && !input.entityId) {
throw new PiiEntityLinkageError(
DECISION_TEMPLATE.replace("%s", input.source.source_id).replace(
"%s",
"policy requires entity linkage (business contact, recorded decision 2026-08-31) — pass entityId (orgnr/rgs/LEI); personal data may only be stored in correspondence with a company/org entity",
),
);
}
const now = Date.now();
const key: PiiIdentityKey = {
key_hash: hash256(`${sourceIdSalt(input.source.source_id)}:${input.identityKey}`),
subject_hash: hash256(input.subjectId),
entity_hash: input.entityId ? hash256(normaliseEntity(input.entityId)) : null,
source_id: input.source.source_id,
tenant_id: input.tenantId,
field: input.field,
artifact_ref: input.artifactRef ?? null,
stored_at: new Date(now).toISOString(),
expires_at: new Date(now + policy.retention_days * 86_400_000).toISOString(),
status: "active",
};
this.registry.push(key);
await this.safeEmit({
event_id: `evt_0pt_pii_retained_${input.source.source_id}_${key.key_hash.slice(0, 12)}`,
event_type: "0pt.pii_data_retained",
capability_id: "0pt.pii_retain_identity",
tenant_id: input.tenantId,
project_id: "",
actor_id: "system",
resource_type: "pii_identity_key",
resource_id: key.key_hash,
risk_level: "L3",
payload: {
source_id: input.source.source_id,
field: input.field,
expires_at: key.expires_at,
entity_linked: key.entity_hash !== null,
},
});
return key;
}
/** Right of access — everything currently held on a subject across sources. */
async accessStatement(subjectHash: string): Promise<PiiAccessStatement> {
const rows = this.registry.filter((r) => r.subject_hash === subjectHash && r.status === "active");
return { subject_hash: subjectHash, rows, generated_at: new Date().toISOString() };
}
/** Right to erasure — tombstone every held row for the subject. */
async erase(request: PiiErasureRequest): Promise<number> {
let count = 0;
for (const row of this.registry) {
if (row.subject_hash === request.subject_hash && row.status === "active") {
row.status = "erased";
count += 1;
}
}
await this.safeEmit({
event_id: `evt_0pt_pii_erased_${request.subject_hash.slice(0, 12)}_${Date.now()}`,
event_type: "0pt.pii_erased",
capability_id: "0pt.pii_erase",
tenant_id: "",
project_id: "",
actor_id: request.actor_id ?? "system",
resource_type: "pii_subject",
resource_id: request.subject_hash,
risk_level: "L3",
payload: { reason: request.reason, row_count: count },
});
return count;
}
/** Storage limitation — purge rows past their retention deadline. */
async enforceRetention(now: Date = new Date()): Promise<{ purged: number }> {
let purged = 0;
const keep: PiiIdentityKey[] = [];
for (const row of this.registry) {
if (row.expires_at <= now.toISOString() && row.status === "active") {
row.status = "purged";
purged += 1;
} else {
keep.push(row);
}
}
if (purged > 0) {
await this.safeEmit({
event_id: `evt_0pt_pii_retention_${now.getTime()}`,
event_type: "0pt.pii_retention_purged",
capability_id: "0pt.pii_enforce_retention",
tenant_id: "",
project_id: "",
actor_id: "system",
resource_type: "pii_identity_key",
resource_id: now.toISOString(),
risk_level: "L3",
payload: { row_count: purged },
});
}
return { purged };
}
/* ------------------------------------------------------------------ *
* Log hygiene — no personuppgifter in logs/metrics/traces
* ------------------------------------------------------------------ */
/** Scrub personnummer, e-mail and Swedish phone numbers from a log line. */
redact(line: string): string {
return line
.replace(/\b\d{8}[-–]\d{4}\b/g, "[PERSONNUMMER]")
.replace(/\b\d{6}[-–]\d{4}\b/g, "[PERSONNUMMER]")
.replace(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g, "[EMAIL]")
.replace(/(?:\+46|0046|0)\s?[1-9][\d\s.-]{7,13}/g, "[TELEFON]");
}
/* ------------------------------------------------------------------ */
private async safeEmit(event: Omit<Aud0Event, "app_id" | "created_at">): Promise<void> {
const full: Aud0Event = {
...event,
app_id: "0pt",
created_at: new Date().toISOString(),
};
try {
await this.aud0.ingestEvent(full);
} catch {
// Audit is important but must never take down a compliance decision or
// a retention pass. The caller's state change has already happened.
}
}
}
function sourceIdSalt(sourceId: string): string {
return `0pt-pii-v1:${sourceId}`;
}
/** Normalise an org/rgs nummer or LEI to canonical digits/letters before hashing. */
function normaliseEntity(entityId: string): string {
return entityId.replace(/[^0-9A-Za-z]/g, "").toUpperCase();
}
/** sha256 hex digest — never the plaintext. */
export function hash256(value: string): string {
return createHash("sha256").update(value, "utf8").digest("hex");
}
+18
View File
@@ -0,0 +1,18 @@
/**
* src/services/quality.ts
* =========================
* Shared singleton for the quality route group (app/api/0pt/quality/*),
* matching src/services/source-registry.ts's precedent: constructing the
* service once here (instead of `new QualityService()` per route module)
* keeps rules/runs/issues registered via one route visible to another
* within the same running process — the same "not even a shared
* process-wide singleton" defect BEFORE_STATE.md §1 found and fixed for the
* Source Registry would otherwise recur here.
*
* This is also the seam the Python engine's quality bridge
* (services/api/workers/quality_bridge.py) calls through over HTTP —
* see app/api/0pt/quality/bridge-run/route.ts.
*/
import { QualityService } from "../quality/quality-service";
export const qualityService = new QualityService();
+33
View File
@@ -0,0 +1,33 @@
import { createAud0Client } from "../clients/aud0-client";
import { resolveDependencyBaseUrl } from "../platform/dependency-mode";
import { SiaxError } from "../platform/errors";
import { createSearchDefinitionRepository } from "../repositories/search-definition-repository";
import { HttpOpenSearchAdapter } from "../search/opensearch-adapter";
import { SearchService } from "../search/search-service";
let singleton: SearchService | undefined;
export function getSearchService(): SearchService {
if (singleton) return singleton;
const baseUrl = resolveDependencyBaseUrl("opensearch", process.env.OPENSEARCH_URL);
if (!baseUrl) {
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "opensearch", "OPENSEARCH_URL is required for the search HTTP surface");
}
singleton = new SearchService(
new HttpOpenSearchAdapter({
baseUrl,
...(process.env.OPENSEARCH_BASIC_AUTH
? { basicAuth: process.env.OPENSEARCH_BASIC_AUTH }
: {}),
}),
{
aud0Client: createAud0Client(),
definitionRepository: createSearchDefinitionRepository(),
},
);
return singleton;
}
export function resetSearchServiceForTests(): void {
singleton = undefined;
}
+238
View File
@@ -0,0 +1,238 @@
/**
* src/services/source-query-service.ts
* =======================================
* 0pt.query_source — bounded, timeout-guarded synchronous query execution
* against a registered Source (source_type = "database",
* adapter_id = "fabric_onelake_sql" today; extensible via `ADAPTER_REGISTRY`).
*
* This is deliberately NOT a general-purpose query engine: it enforces a
* max-rows and max-time ceiling on every call and is meant for small,
* interactive/preview-style reads. Anything heavier (large exports, long
* scans, scheduled pulls) must go through N0D as a job — see
* docs/P0_QUERY_SOURCE_FLOW.md — the same "synchronous vs. N0D job" split
* `quality-check-service.ts` uses for quality checks.
*
* Reuses the existing SourceService (src/services/source-service.ts) purely
* through its public `getSource()` — this file does not modify that class or
* the Source/CreateSourceInput types, which already carry the `adapter_id`
* field this capability depends on.
*/
import type { Source } from "../types/source";
import { SourceService } from "./source-service";
import { PiiGateService } from "./pii-gate-service";
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import { FabricOneLakeAdapter, type FabricOneLakeConnectionConfig, type AdapterQueryResult } from "../adapters/fabric-onelake-adapter";
/** Default bound applied when the caller does not specify one. */
export const SOURCE_QUERY_DEFAULT_MAX_ROWS = 1000;
/** Absolute ceiling — a caller cannot raise max_rows above this on the synchronous path. */
export const SOURCE_QUERY_HARD_MAX_ROWS = 10_000;
/** Default timeout applied when the caller does not specify one. */
export const SOURCE_QUERY_DEFAULT_TIMEOUT_MS = 10_000;
/** Absolute ceiling — a caller cannot raise timeout_ms above this on the synchronous path. */
export const SOURCE_QUERY_HARD_MAX_TIMEOUT_MS = 30_000;
/** Adapter ids supported by the synchronous query path today. */
export type SupportedQueryAdapterId = "fabric_onelake_sql";
export class SourceQueryTimeoutError extends Error {}
export class SourceQueryUnsupportedError extends Error {}
export interface QuerySourceInput {
source_id: string;
query: string;
params?: Record<string, unknown>;
tenant_id: string;
project_id: string;
actor_id?: string;
correlation_id?: string;
/** Clamped to [1, SOURCE_QUERY_HARD_MAX_ROWS]. Defaults to SOURCE_QUERY_DEFAULT_MAX_ROWS. */
max_rows?: number;
/** Clamped to [1, SOURCE_QUERY_HARD_MAX_TIMEOUT_MS]. Defaults to SOURCE_QUERY_DEFAULT_TIMEOUT_MS. */
timeout_ms?: number;
/**
* Connection config for the source's adapter. Never persisted by this
* service and never defaulted from an env var — tenant-agnostic by
* construction, same as the adapter itself. Today only the
* "fabric_onelake_sql" shape is supported.
*/
connection_config: FabricOneLakeConnectionConfig;
}
export interface QuerySourceResult {
source_id: string;
rows: Record<string, unknown>[];
row_count: number;
truncated: boolean;
duration_ms: number;
bounds: { max_rows: number; timeout_ms: number };
}
function clamp(value: number, min: number, max: number): number {
if (!Number.isFinite(value)) return min;
return Math.min(Math.max(value, min), max);
}
export class SourceQueryService {
private sourceService: SourceService;
private aud0: Aud0Client;
/** GDPR guardrail — on by default, can be shared via opts so callers use one registry. */
private piiGate: PiiGateService;
constructor(opts?: { sourceService?: SourceService; aud0BaseUrl?: string; piiGate?: PiiGateService }) {
// Accepts an existing SourceService so callers (e.g. the API route) share
// one in-memory registry instead of each constructing an empty one.
this.sourceService = opts?.sourceService ?? new SourceService();
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.piiGate = opts?.piiGate ?? new PiiGateService({ aud0BaseUrl: opts?.aud0BaseUrl });
}
async querySource(input: QuerySourceInput): Promise<QuerySourceResult> {
const startedAt = Date.now();
// GDPR fail-closed: personal-data sources (self-flagged or known directory
// domains such as ratsit.se) may not be processed without a PiiPolicy.
// Runs BEFORE the capability/type gate so the lawful-basis error is the
// one the caller sees for personal data.
const preflight = await this.sourceService.getSource(input.source_id);
await this.piiGate.assertCanStore(preflight, [], input.tenant_id);
const source = await this.getQueryableSource(input.source_id);
const maxRows = clamp(input.max_rows ?? SOURCE_QUERY_DEFAULT_MAX_ROWS, 1, SOURCE_QUERY_HARD_MAX_ROWS);
const timeoutMs = clamp(input.timeout_ms ?? SOURCE_QUERY_DEFAULT_TIMEOUT_MS, 1, SOURCE_QUERY_HARD_MAX_TIMEOUT_MS);
const correlationId = input.correlation_id ?? `corr_0pt_query_source_${input.source_id}_${startedAt}`;
const adapter = this.buildAdapter(source, input.connection_config);
try {
const { rows, durationMs } = await this.runWithTimeout(adapter.query(input.query, input.params), timeoutMs);
const truncated = rows.length > maxRows;
const boundedRows = truncated ? rows.slice(0, maxRows) : rows;
await this.emitQueriedEvent({
source,
input,
correlationId,
rowCount: boundedRows.length,
truncated,
durationMs,
ok: true,
});
return {
source_id: source.source_id,
rows: boundedRows,
row_count: boundedRows.length,
truncated,
duration_ms: durationMs,
bounds: { max_rows: maxRows, timeout_ms: timeoutMs },
};
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
await this.emitQueriedEvent({
source,
input,
correlationId,
rowCount: 0,
truncated: false,
durationMs: Date.now() - startedAt,
ok: false,
error: message,
});
throw err instanceof Error ? err : new Error(message);
} finally {
// Each call opens its own short-lived adapter/pool (see buildAdapter) —
// always release it, whether the query succeeded, failed, or timed out.
await adapter.close().catch(() => {});
}
}
private async getQueryableSource(sourceId: string): Promise<Source> {
const source = await this.sourceService.getSource(sourceId);
if (source.source_type !== "database") {
throw new SourceQueryUnsupportedError(
`Source ${sourceId} has source_type "${source.source_type}", not "database" — the synchronous 0pt.query_source path only ` +
`supports database sources. Read non-database sources through a N0D job instead.`,
);
}
if (source.adapter_id !== "fabric_onelake_sql") {
throw new SourceQueryUnsupportedError(
`Source ${sourceId} has adapter_id "${source.adapter_id ?? "(none)"}" — the synchronous 0pt.query_source path currently only ` +
`supports adapter_id "fabric_onelake_sql". Register the source with that adapter_id, or run the read through N0D as a job instead.`,
);
}
return source;
}
private buildAdapter(source: Source, connectionConfig: FabricOneLakeConnectionConfig): FabricOneLakeAdapter {
if (!connectionConfig) {
throw new Error(`Source ${source.source_id}: connection_config is required to query a fabric_onelake_sql source`);
}
return new FabricOneLakeAdapter(connectionConfig);
}
/**
* Races the adapter query against a timeout. Note this bounds *waiting*
* time on the caller's side — it does not cancel an in-flight TDS request
* on timeout (the `mssql` driver has no cooperative cancellation hook used
* here), so a timed-out query may still complete against the warehouse
* after this call has already thrown. That is an accepted trade-off for a
* synchronous, best-effort preview path; anything where that matters
* belongs on the N0D job path instead.
*/
private async runWithTimeout<T>(promise: Promise<AdapterQueryResult<T>>, timeoutMs: number): Promise<AdapterQueryResult<T>> {
let timer: ReturnType<typeof setTimeout> | undefined;
const timeout = new Promise<never>((_, reject) => {
timer = setTimeout(() => {
reject(
new SourceQueryTimeoutError(
`source query exceeded the bounded synchronous timeout of ${timeoutMs}ms — run this query through N0D as a job instead of the synchronous 0pt.query_source path`,
),
);
}, timeoutMs);
});
try {
return await Promise.race([promise, timeout]);
} finally {
clearTimeout(timer);
}
}
private async emitQueriedEvent(args: {
source: Source;
input: QuerySourceInput;
correlationId: string;
rowCount: number;
truncated: boolean;
durationMs: number;
ok: boolean;
error?: string;
}): Promise<void> {
const { source, input, correlationId, rowCount, truncated, durationMs, ok, error } = args;
await this.aud0.ingestEvent({
event_id: `evt_0pt_source_queried_${source.source_id}_${Date.now()}`,
event_type: "0pt.source_queried",
tenant_id: input.tenant_id,
project_id: input.project_id,
actor_id: input.actor_id ?? "system",
app_id: "0pt",
capability_id: "0pt.query_source",
resource_type: "source",
resource_id: source.source_id,
risk_level: "L4",
payload: {
adapter_id: source.adapter_id,
ok,
row_count: rowCount,
truncated,
duration_ms: durationMs,
query_preview: input.query.slice(0, 200),
...(error ? { error } : {}),
},
created_at: new Date().toISOString(),
correlation_id: correlationId,
});
}
}
+156
View File
@@ -0,0 +1,156 @@
import type {
SourceRatePolicy,
SourceRatePolicyStatus,
CreateSourceRatePolicyInput,
} from "../types/source-rate-policy";
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import { SourceService } from "./source-service";
interface SourceRatePolicyRecord {
source_rate_policy_id: string;
tenant_id: string;
project_id: string;
source_id: string;
rate_limit_per_minute: number;
crawl_budget: number;
compliance_notes: string | null;
domain: string | null;
robots_ok: boolean | null;
access: string | null;
status: string;
correlation_id: string | null;
created_at: string;
updated_at: string | null;
created_by: string | null;
}
/**
* Owns 0pt.source_rate_policy.schema.v1 -- the genuine ownership repair:
* source/crawl-domain rate limiting is 0pt's own compliance policy, moved
* OUT of N0D's ComputePolicy.sourceRatePolicy (deprecated there per
* ADR-0030's audit). See docs/ICEBERG_WRITE_BOUNDARY.md.
*/
export class SourceRatePolicyService {
private policies: Map<string, SourceRatePolicyRecord> = new Map();
private aud0: Aud0Client;
private sourceService: SourceService;
constructor(opts?: { aud0BaseUrl?: string; sourceService?: SourceService }) {
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.sourceService = opts?.sourceService ?? new SourceService({ aud0BaseUrl: opts?.aud0BaseUrl });
}
async createSourceRatePolicy(input: CreateSourceRatePolicyInput): Promise<SourceRatePolicy> {
if (this.policies.has(input.source_rate_policy_id)) {
throw new Error(`Source rate policy ${input.source_rate_policy_id} already exists`);
}
if (input.rate_limit_per_minute <= 0) {
throw new Error("rate_limit_per_minute must be > 0");
}
if (input.crawl_budget <= 0) {
throw new Error("crawl_budget must be > 0");
}
// Verify the source exists -- a rate policy with no source is meaningless.
try {
await this.sourceService.getSource(input.source_id);
} catch {
throw new Error(`Source ${input.source_id} not found`);
}
const now = new Date().toISOString();
const record: SourceRatePolicyRecord = {
source_rate_policy_id: input.source_rate_policy_id,
tenant_id: input.tenant_id,
project_id: input.project_id,
source_id: input.source_id,
rate_limit_per_minute: input.rate_limit_per_minute,
crawl_budget: input.crawl_budget,
compliance_notes: input.compliance_notes ?? null,
domain: input.domain ?? null,
robots_ok: input.robots_ok ?? null,
access: input.access ?? null,
status: input.status ?? "active",
correlation_id: input.correlation_id ?? null,
created_at: now,
updated_at: null,
created_by: input.created_by ?? null,
};
this.policies.set(input.source_rate_policy_id, record);
// 0pt.source_rate_policy_created -- docs/11-lakehouse-write-plan.md event list,
// docs/ICEBERG_WRITE_BOUNDARY.md. Gives AUD0/CL0UD an evented trail of 0pt's
// crawl-compliance posture without polling 0pt's DB.
await this.aud0.ingestEvent({
event_id: `evt_0pt_source_rate_policy_created_${input.source_rate_policy_id}`,
event_type: "0pt.source_rate_policy_created",
tenant_id: input.tenant_id,
project_id: input.project_id,
actor_id: input.created_by ?? "system",
app_id: "0pt",
capability_id: "0pt.create_source_rate_policy",
resource_type: "source_rate_policy",
resource_id: input.source_rate_policy_id,
risk_level: "L2",
payload: {
source_id: input.source_id,
rate_limit_per_minute: input.rate_limit_per_minute,
crawl_budget: input.crawl_budget,
},
created_at: now,
correlation_id: input.correlation_id ?? "",
});
return this.toPolicy(record);
}
// P014: `tenantId` is optional so existing internal callers that already
// resolved a policy from a tenant-scoped list (or are trusted, in-process
// callers) are unaffected — but the HTTP route now always passes it, which
// closes a real gap: this lookup used to be reachable by rate-policy ID
// alone with NO tenant check anywhere, so any authenticated caller could
// read another tenant's crawl-budget/rate-limit/compliance-notes record
// just by guessing or enumerating IDs. A mismatched tenantId is reported
// identically to "not found" (no existence oracle), matching the
// convention SourceRegistryService.assertSourceTenant already established.
async getSourceRatePolicy(sourceRatePolicyId: string, tenantId?: string): Promise<SourceRatePolicy> {
const record = this.policies.get(sourceRatePolicyId);
if (!record || (tenantId && record.tenant_id !== tenantId)) {
throw new Error(`Source rate policy ${sourceRatePolicyId} not found`);
}
return this.toPolicy(record);
}
async listSourceRatePolicies(filter?: {
tenant_id?: string;
project_id?: string;
source_id?: string;
status?: SourceRatePolicyStatus;
}): Promise<SourceRatePolicy[]> {
let results = Array.from(this.policies.values());
if (filter?.tenant_id) results = results.filter((r) => r.tenant_id === filter.tenant_id);
if (filter?.project_id) results = results.filter((r) => r.project_id === filter.project_id);
if (filter?.source_id) results = results.filter((r) => r.source_id === filter.source_id);
if (filter?.status) results = results.filter((r) => r.status === filter.status);
return results.map((r) => this.toPolicy(r));
}
private toPolicy(r: SourceRatePolicyRecord): SourceRatePolicy {
return {
source_rate_policy_id: r.source_rate_policy_id,
tenant_id: r.tenant_id,
project_id: r.project_id,
source_id: r.source_id,
rate_limit_per_minute: r.rate_limit_per_minute,
crawl_budget: r.crawl_budget,
compliance_notes: r.compliance_notes ?? undefined,
domain: r.domain ?? undefined,
robots_ok: r.robots_ok ?? undefined,
access: (r.access as SourceRatePolicy["access"]) ?? undefined,
status: r.status as SourceRatePolicyStatus,
correlation_id: r.correlation_id ?? undefined,
created_at: r.created_at,
updated_at: r.updated_at ?? undefined,
created_by: r.created_by ?? undefined,
};
}
}
+511
View File
@@ -0,0 +1,511 @@
/**
* src/services/source-registry-service.ts
* ========================================
* The P0 Source Registry capability surface (Master-DoD §10):
*
* source.register -> delegates to SourceService.createSource (owner of
* 0pt.source.schema.v1) + optional rights/provenance
* source.query -> listSources + rights/purpose filtering
* source.describe -> full SourceDescriptor (base + rights + provenance
* + health + freshness + rate policy)
* source.health -> record + aggregate health observations
* source.rate-policy -> already owned by SourceRatePolicyService; composed
*
* Tenant isolation: every read path REQUIRES a tenant_id and filters on it —
* a source registered by tenant A is invisible to tenant B (tested in
* tests/p0/source-registry.test.ts, incl. the cross-tenant attack case §50).
* `getHealth`/`getFreshness` now take tenant_id directly and run the same
* `assertSourceTenant` gate every other accessor does — they are exposed as
* standalone HTTP routes now (app/api/0pt/sources/[sourceId]/health,
* .../freshness), so they can no longer rely on an earlier caller having
* already checked tenancy on their behalf.
*
* Purpose enforcement: `assertPurposeAllowed` is the single hook other 0pt
* capabilities call before using a source for a given purpose (SearchService
* and DatasetService both do). No rights record ⇒ fail closed (deny).
*
* Persistence: rights/provenance/health/freshness now live behind
* SourceRegistryRepository (src/repositories/source-registry-repository.ts)
* — real Postgres (`opt_source_rights`/`opt_source_provenance`/
* `opt_source_health_checks`/`opt_source_freshness`,
* sql/002_source_registry.sql) when DATABASE_URL is configured, in-memory
* Maps otherwise. `resolveStoreMode` still fails production boot hard with
* no DATABASE_URL (Master-DoD §3) — the difference from before is that a
* configured production now actually WORKS (opt_sources/etc. exist), not
* just fails safely.
*/
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import { SiaxError } from "../platform/errors";
import {
createSourceRegistryRepository,
type SourceRegistryRepository,
} from "../repositories/source-registry-repository";
import { SourceService, type SourceServiceOptions } from "./source-service";
import { SourceRatePolicyService } from "./source-rate-policy-service";
import type { CreateSourceInput, Source } from "../types/source";
import type {
SourceAuthorization,
SourceClassification,
SourceDescriptor,
SourceFreshness,
SourceHealth,
SourceHealthCheck,
SourceHealthStatus,
SourceProvenance,
SourceRights,
SourceSchedule,
} from "../types/source-registry";
export interface RegisterSourceInput extends CreateSourceInput {
rights?: Omit<SourceRights, "source_id" | "tenant_id">;
provenance?: Omit<SourceProvenance, "source_id" | "tenant_id" | "updated_at">;
authorization?: Omit<SourceAuthorization, "source_id" | "tenant_id">;
schedule?: Omit<SourceSchedule, "source_id" | "tenant_id" | "updated_at">;
}
/**
* Actor ids that may never appear as an authorization's `granted_by`.
* P060 requires an accountable HUMAN decision behind a source going live;
* a service identity granting itself access is exactly the rubber stamp the
* gate exists to prevent.
*/
const NON_ACCOUNTABLE_ACTORS = /^(system|svc_|service:|robot|automation|anonymous)/i;
export class SourceRegistryService {
private readonly repo: SourceRegistryRepository;
private readonly aud0: Aud0Client;
readonly sources: SourceService;
readonly ratePolicies: SourceRatePolicyService;
constructor(opts?: {
aud0BaseUrl?: string;
/** Explicit DATABASE_URL override — mainly for tests. Falls back to resolveStoreMode's own env lookup when omitted. */
databaseUrl?: string;
sourceService?: SourceService;
ratePolicyService?: SourceRatePolicyService;
/** Inject a repository directly (tests) — bypasses resolveStoreMode entirely. */
repository?: SourceRegistryRepository;
}) {
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.repo = opts?.repository ?? createSourceRegistryRepository(opts?.databaseUrl);
const sourceServiceOpts: SourceServiceOptions = {
aud0BaseUrl: opts?.aud0BaseUrl,
databaseUrl: opts?.databaseUrl,
};
this.sources = opts?.sourceService ?? new SourceService(sourceServiceOpts);
this.ratePolicies =
opts?.ratePolicyService ??
new SourceRatePolicyService({ aud0BaseUrl: opts?.aud0BaseUrl, sourceService: this.sources });
}
// -------------------------------------------------------------------------
// source.register
// -------------------------------------------------------------------------
async registerSource(input: RegisterSourceInput): Promise<SourceDescriptor> {
const { rights, provenance, authorization, schedule, ...base } = input;
// P060: registration NEVER activates. A caller asking for status "active"
// gets the source created "pending" and must go through activateSource,
// which checks the classification. Silently honouring `status: "active"`
// here would leave the gate wide open to its most obvious bypass.
const requestedActive = base.status === "active";
const source = await this.sources.createSource({
...base,
status: requestedActive ? "pending" : base.status,
});
if (rights) {
await this.setRights({ ...rights, source_id: source.source_id, tenant_id: source.tenant_id });
}
if (provenance) {
await this.setProvenance({
...provenance,
source_id: source.source_id,
tenant_id: source.tenant_id,
updated_at: new Date().toISOString(),
});
}
if (authorization) {
await this.setAuthorization({
...authorization,
source_id: source.source_id,
tenant_id: source.tenant_id,
});
}
if (schedule) {
await this.setSchedule({
...schedule,
source_id: source.source_id,
tenant_id: source.tenant_id,
updated_at: new Date().toISOString(),
});
}
// A registration that arrived fully classified AND asked to be active is
// activated here — through the same gate, not around it.
if (requestedActive) {
await this.activateSource(source.source_id, source.tenant_id, base.created_by);
}
return this.describeSource(source.source_id, source.tenant_id);
}
// -------------------------------------------------------------------------
// P060 authorization / schedule
// -------------------------------------------------------------------------
async setAuthorization(auth: SourceAuthorization): Promise<SourceAuthorization> {
await this.assertSourceTenant(auth.source_id, auth.tenant_id);
if (!auth.granted_by || NON_ACCOUNTABLE_ACTORS.test(auth.granted_by)) {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.source_registry",
`authorization.granted_by must name an accountable actor, not "${auth.granted_by}"`,
);
}
await this.repo.setAuthorization(auth);
await this.emit("0pt.source_authorization_set", auth.tenant_id, auth.source_id, {
basis: auth.basis,
granted_by: auth.granted_by,
expires_at: auth.expires_at,
evidence_ref: auth.evidence_ref,
});
return auth;
}
async getAuthorization(sourceId: string, tenantId: string): Promise<SourceAuthorization | undefined> {
await this.assertSourceTenant(sourceId, tenantId);
return this.repo.getAuthorization(sourceId);
}
async setSchedule(schedule: SourceSchedule): Promise<SourceSchedule> {
await this.assertSourceTenant(schedule.source_id, schedule.tenant_id);
if (schedule.interval_seconds !== undefined && schedule.interval_seconds <= 0) {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.source_registry",
"schedule.interval_seconds must be positive",
);
}
await this.repo.setSchedule(schedule);
await this.emit("0pt.source_schedule_set", schedule.tenant_id, schedule.source_id, {
mode: schedule.mode,
interval_seconds: schedule.interval_seconds,
enabled: schedule.enabled,
});
return schedule;
}
async getSchedule(sourceId: string, tenantId: string): Promise<SourceSchedule | undefined> {
await this.assertSourceTenant(sourceId, tenantId);
return this.repo.getSchedule(sourceId);
}
// -------------------------------------------------------------------------
// P060 classification + activation gate
// -------------------------------------------------------------------------
/**
* Is this source classified? A source is classified when ALL of the
* following are on record:
* - an accountable owner,
* - an unexpired authorization from an accountable actor,
* - rights: a license id AND at least one allowed purpose,
* - provenance: how the data is obtained.
*
* Everything else in the registry (schedule, schema, checkpoint) describes
* HOW a source runs, not WHETHER it may — those are deliberately not part
* of the gate, and that choice is documented here rather than left implicit.
*/
async classifySource(
sourceId: string,
tenantId: string,
now: Date = new Date(),
): Promise<SourceClassification> {
const source = await this.assertSourceTenant(sourceId, tenantId);
const missing: string[] = [];
if (!source.owner_id) missing.push("owner_id is not set");
const auth = await this.repo.getAuthorization(sourceId);
if (!auth) {
missing.push("no authorization on record");
} else {
if (!auth.granted_by || NON_ACCOUNTABLE_ACTORS.test(auth.granted_by)) {
missing.push(`authorization.granted_by "${auth.granted_by}" is not an accountable actor`);
}
if (auth.expires_at && new Date(auth.expires_at).getTime() <= now.getTime()) {
missing.push(`authorization expired at ${auth.expires_at}`);
}
}
const rights = await this.repo.getRights(sourceId);
if (!rights) {
missing.push("no rights record (license/purpose) on record");
} else {
if (!rights.license_id) missing.push("rights.license_id is empty");
if (rights.allowed_purposes.length === 0) missing.push("rights.allowed_purposes is empty");
}
const provenance = await this.repo.getProvenance(sourceId);
if (!provenance) missing.push("no provenance record (sourcing method) on record");
return { source_id: sourceId, classified: missing.length === 0, missing };
}
/**
* P060 DoD: an UNCLASSIFIED source cannot be activated. Fail-closed —
* every missing fact is named at once, and nothing is written when the
* check fails.
*/
async activateSource(sourceId: string, tenantId: string, actorId?: string): Promise<Source> {
const classification = await this.classifySource(sourceId, tenantId);
if (!classification.classified) {
await this.emit("0pt.source_activation_denied", tenantId, sourceId, {
missing: classification.missing,
});
throw new SiaxError(
"AUTHORIZATION_DENIED",
"0pt.source_registry",
`source ${sourceId} is not classified and cannot be activated: ${classification.missing.join("; ")}`,
);
}
const source = await this.sources.updateStatus(sourceId, "active", actorId);
await this.emit("0pt.source_activated", tenantId, sourceId, { activated_by: actorId ?? null });
return source;
}
/** Take a source out of service. Always allowed — deactivating is never gated. */
async deactivateSource(
sourceId: string,
tenantId: string,
status: "paused" | "archived" | "deprecated" = "paused",
actorId?: string,
): Promise<Source> {
await this.assertSourceTenant(sourceId, tenantId);
const source = await this.sources.updateStatus(sourceId, status, actorId);
await this.emit("0pt.source_deactivated", tenantId, sourceId, { status });
return source;
}
// -------------------------------------------------------------------------
// rights / license / purpose metadata
// -------------------------------------------------------------------------
async setRights(rights: SourceRights): Promise<SourceRights> {
await this.assertSourceTenant(rights.source_id, rights.tenant_id);
await this.repo.setRights(rights);
await this.emit("0pt.source_rights_set", rights.tenant_id, rights.source_id, {
license_id: rights.license_id,
allowed_purposes: rights.allowed_purposes,
redistribution: rights.redistribution,
});
return rights;
}
async getRights(sourceId: string, tenantId: string): Promise<SourceRights | undefined> {
await this.assertSourceTenant(sourceId, tenantId);
return this.repo.getRights(sourceId);
}
/**
* The purpose gate other capabilities call before using a source. FAIL
* CLOSED: no rights record, or a purpose outside the allowlist, throws
* AUTHORIZATION_DENIED.
*/
async assertPurposeAllowed(sourceId: string, tenantId: string, purpose: string): Promise<SourceRights> {
await this.assertSourceTenant(sourceId, tenantId);
const rights = await this.repo.getRights(sourceId);
if (!rights) {
throw new SiaxError(
"AUTHORIZATION_DENIED",
"0pt.source_registry",
`source ${sourceId} has no rights record — purpose "${purpose}" denied (fail closed)`,
);
}
if (!rights.allowed_purposes.includes(purpose)) {
throw new SiaxError(
"AUTHORIZATION_DENIED",
"0pt.source_registry",
`purpose "${purpose}" is not in source ${sourceId}'s allowed purposes`,
);
}
return rights;
}
// -------------------------------------------------------------------------
// provenance / freshness
// -------------------------------------------------------------------------
async setProvenance(prov: SourceProvenance): Promise<SourceProvenance> {
await this.assertSourceTenant(prov.source_id, prov.tenant_id);
await this.repo.setProvenance(prov);
await this.emit("0pt.source_provenance_set", prov.tenant_id, prov.source_id, {
method: prov.method,
authority: prov.authority,
cadence: prov.cadence,
});
return prov;
}
async getProvenance(sourceId: string, tenantId: string): Promise<SourceProvenance | undefined> {
await this.assertSourceTenant(sourceId, tenantId);
return this.repo.getProvenance(sourceId);
}
/** Record a successful data observation (drives freshness). */
async recordSuccess(sourceId: string, tenantId: string, atIso?: string): Promise<void> {
await this.assertSourceTenant(sourceId, tenantId);
await this.repo.recordSuccess(sourceId, tenantId, atIso ?? new Date().toISOString());
}
async getFreshness(sourceId: string, tenantId: string, now: Date = new Date()): Promise<SourceFreshness> {
await this.assertSourceTenant(sourceId, tenantId);
const prov = await this.repo.getProvenance(sourceId);
const last = await this.repo.getLastSuccess(sourceId);
const sla = prov?.freshness_sla_hours;
if (!last) {
return { source_id: sourceId, freshness_sla_hours: sla, fresh: sla === undefined };
}
const ageHours = (now.getTime() - new Date(last).getTime()) / 3_600_000;
return {
source_id: sourceId,
last_success_at: last,
freshness_sla_hours: sla,
age_hours: Math.round(ageHours * 100) / 100,
fresh: sla === undefined ? true : ageHours <= sla,
};
}
// -------------------------------------------------------------------------
// source.health
// -------------------------------------------------------------------------
async recordHealthCheck(check: SourceHealthCheck): Promise<SourceHealth> {
await this.assertSourceTenant(check.source_id, check.tenant_id);
await this.repo.appendHealthCheck(check);
if (check.status === "up") {
await this.recordSuccess(check.source_id, check.tenant_id, check.checked_at);
}
return this.getHealth(check.source_id, check.tenant_id);
}
async getHealth(sourceId: string, tenantId: string): Promise<SourceHealth> {
await this.assertSourceTenant(sourceId, tenantId);
// Newest-first (repository contract) — index 0 is the most recent check.
const list = await this.repo.listHealthChecks(sourceId);
const last = list[0];
let consecutiveFailures = 0;
for (const check of list) {
if (check.status === "down" || check.status === "degraded") consecutiveFailures++;
else break;
}
const status: SourceHealthStatus = last?.status ?? "unknown";
return { source_id: sourceId, status, last_check: last, consecutive_failures: consecutiveFailures };
}
// -------------------------------------------------------------------------
// source.query / source.describe
// -------------------------------------------------------------------------
async querySources(filter: {
tenant_id: string;
project_id?: string;
source_type?: Source["source_type"];
status?: Source["status"];
/** Only sources whose rights allowlist includes this purpose. */
purpose?: string;
/** Only sources with at most this redistribution openness. */
redistribution?: SourceRights["redistribution"];
}): Promise<Source[]> {
if (!filter.tenant_id) {
throw new SiaxError("INVALID_REQUEST", "0pt.source_registry", "tenant_id is required on source.query");
}
let results = await this.sources.listSources({
tenant_id: filter.tenant_id,
project_id: filter.project_id,
source_type: filter.source_type,
status: filter.status,
});
if (filter.purpose !== undefined || filter.redistribution !== undefined) {
const rightsEntries = await Promise.all(
results.map(async (s) => [s.source_id, await this.repo.getRights(s.source_id)] as const),
);
const rightsBySource = new Map(rightsEntries);
if (filter.purpose !== undefined) {
results = results.filter((s) => {
const rights = rightsBySource.get(s.source_id);
return rights !== undefined && rights.allowed_purposes.includes(filter.purpose as string);
});
}
if (filter.redistribution !== undefined) {
results = results.filter(
(s) => rightsBySource.get(s.source_id)?.redistribution === filter.redistribution,
);
}
}
return results;
}
async describeSource(sourceId: string, tenantId: string): Promise<SourceDescriptor> {
const source = await this.assertSourceTenant(sourceId, tenantId);
const [policies, rights, provenance, health, freshness, authorization, schedule, classification] =
await Promise.all([
this.ratePolicies.listSourceRatePolicies({ tenant_id: tenantId, source_id: sourceId }),
this.repo.getRights(sourceId),
this.repo.getProvenance(sourceId),
this.getHealth(sourceId, tenantId),
this.getFreshness(sourceId, tenantId),
this.repo.getAuthorization(sourceId),
this.repo.getSchedule(sourceId),
this.classifySource(sourceId, tenantId),
]);
return {
source,
rights,
provenance,
health,
freshness,
rate_policy: policies[0],
authorization,
schedule,
classification,
};
}
// -------------------------------------------------------------------------
// internals
// -------------------------------------------------------------------------
/**
* Tenant gate for every read/write: resolves the source AND verifies the
* caller's tenant owns it. Cross-tenant probes get NOT_FOUND (no existence
* oracle), same 404-not-403 posture as ST0RE/N0D.
*/
private async assertSourceTenant(sourceId: string, tenantId: string): Promise<Source> {
if (!tenantId) {
throw new SiaxError("INVALID_REQUEST", "0pt.source_registry", "tenant_id is required");
}
let source: Source;
try {
source = await this.sources.getSource(sourceId);
} catch {
throw new SiaxError("NOT_FOUND", "0pt.source_registry", `source ${sourceId} not found`);
}
if (source.tenant_id !== tenantId) {
throw new SiaxError("NOT_FOUND", "0pt.source_registry", `source ${sourceId} not found`);
}
return source;
}
private async emit(
eventType: string,
tenantId: string,
sourceId: string,
payload: Record<string, unknown>,
): Promise<void> {
await this.aud0.ingestEvent({
event_id: `evt_${eventType}_${sourceId}_${Date.now()}`,
event_type: eventType,
tenant_id: tenantId,
project_id: "p_platform",
actor_id: "svc_0pt",
app_id: "0pt",
capability_id: eventType.replace(/^0pt\./, "0pt."),
resource_type: "source",
resource_id: sourceId,
risk_level: "L2",
payload,
created_at: new Date().toISOString(),
});
}
}
+46
View File
@@ -0,0 +1,46 @@
/**
* src/services/source-registry.ts
* =================================
* Shared singleton instances for the source-registry route group
* (app/api/0pt/sources/*), matching src/services/lakehouse-mapping-registry.ts's
* precedent.
*
* BEFORE_STATE.md §1 flagged this as materially worse than plain
* in-memory-per-module: each route file previously constructed its own
* `new SourceService()` (and `new SourceRatePolicyService()`), so a source
* registered via one route module was invisible to another within the same
* running process — "not even a shared process-wide singleton". Once
* DATABASE_URL is configured this stops mattering (state lives in Postgres,
* genuinely shared across processes/deploys — see src/repositories/*), but
* it still matters for the in-memory (no DATABASE_URL) dev/test mode, where
* state must be shared within one process to be usable at all. Constructing
* all three here once, wired together, and importing them everywhere fixes
* both call sites this repo already had (sources/rate-policies/*,
* sources/[sourceId]/query) plus every new route this pass adds.
*/
import { SourceService } from "./source-service";
import { SourceRatePolicyService } from "./source-rate-policy-service";
import { SourceRegistryService } from "./source-registry-service";
export const sourceService = new SourceService();
export const sourceRatePolicyService = new SourceRatePolicyService({ sourceService });
export const sourceRegistryService = new SourceRegistryService({
sourceService,
ratePolicyService: sourceRatePolicyService,
});
/**
* P060-P068 ingestion plane singletons. Same reasoning as above: the schema
* registry, checkpoint store, ledger, quarantine and lineage projection all
* share ONE repository so the in-memory dev/test mode is coherent within a
* process, and one `resolveStoreMode` decision in production.
*/
import { createIngestionRepository } from "../repositories/ingestion-repository";
import { SchemaRegistry } from "../schema/schema-registry";
import { LineageStore } from "../lineage/lineage-store";
import { IngestionRunner } from "../ingestion/ingestion-runner";
export const ingestionRepository = createIngestionRepository();
export const schemaRegistry = new SchemaRegistry(ingestionRepository);
export const lineageStore = new LineageStore(ingestionRepository);
export const ingestionRunner = new IngestionRunner(ingestionRepository, schemaRegistry, lineageStore);
+114
View File
@@ -0,0 +1,114 @@
/**
* src/services/source-service.ts
* ===============================
* Owns 0pt.source.schema.v1 — the per-tenant registry of upstream data
* sources (databases, crawl domains, APIs, file drops). This is the registry
* `SourceQueryService.querySource()` (src/services/source-query-service.ts)
* and `SourceRatePolicyService` (src/services/source-rate-policy-service.ts)
* both look a `source_id` up against — see docs/P0_QUERY_SOURCE_FLOW.md.
*
* Persistence: storage now lives behind SourceRepository
* (src/repositories/source-repository.ts) — real Postgres (`opt_sources`,
* sql/002_source_registry.sql) when DATABASE_URL is configured, an
* in-memory Map otherwise (non-production only; resolveStoreMode fails hard
* in production with no URL). This service still owns the business rules
* (validation, tenant-agnostic CRUD orchestration) and the AUD0 emission —
* only the storage moved.
*
* Callers that need a shared view across route modules should use the
* shared singleton in src/services/source-registry.ts rather than
* constructing a fresh `new SourceService()` per module — see that file's
* doc comment for the "each route owned its own registry" history this
* fixes.
*/
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import {
createSourceRepository,
type SourceRepository,
type SourceListFilter,
} from "../repositories/source-repository";
import type { CreateSourceInput, Source, SourceStatus, SourceType } from "../types/source";
export interface SourceServiceOptions {
aud0BaseUrl?: string;
/** Explicit DATABASE_URL override — mainly for tests pointing at a specific Postgres instance. Falls back to resolveStoreMode's own env lookup when omitted. */
databaseUrl?: string;
/** Inject a repository directly (tests) — bypasses resolveStoreMode entirely. */
repository?: SourceRepository;
}
export class SourceService {
private readonly repo: SourceRepository;
private readonly aud0: Aud0Client;
constructor(opts?: SourceServiceOptions) {
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.repo = opts?.repository ?? createSourceRepository(opts?.databaseUrl);
}
async createSource(input: CreateSourceInput): Promise<Source> {
const source = await this.repo.create(input);
await this.aud0.ingestEvent({
event_id: `evt_0pt_source_created_${source.source_id}`,
event_type: "0pt.source_created",
tenant_id: source.tenant_id,
project_id: source.project_id,
actor_id: source.created_by ?? "system",
app_id: "0pt",
capability_id: "0pt.create_source",
resource_type: "source",
resource_id: source.source_id,
risk_level: "L2",
payload: {
source_type: source.source_type,
name: source.name,
adapter_id: source.adapter_id,
access: source.access,
},
created_at: source.created_at,
correlation_id: source.correlation_id,
});
return source;
}
async getSource(sourceId: string): Promise<Source> {
return this.repo.get(sourceId);
}
/**
* P060: the ONLY status transition path. Callers do not get a generic
* update method, because status is what the activation gate protects —
* SourceRegistryService.activateSource is the only caller that moves a
* source to "active", and it runs the classification check first.
*/
async updateStatus(sourceId: string, status: SourceStatus, actorId?: string): Promise<Source> {
const source = await this.repo.updateStatus(sourceId, status);
await this.aud0.ingestEvent({
event_id: `evt_0pt_source_status_${source.source_id}_${Date.now()}`,
event_type: "0pt.source_status_changed",
tenant_id: source.tenant_id,
project_id: source.project_id,
actor_id: actorId ?? source.owner_id ?? "system",
app_id: "0pt",
capability_id: "0pt.update_source_status",
resource_type: "source",
resource_id: source.source_id,
risk_level: "L2",
payload: { status: source.status, name: source.name },
created_at: source.updated_at ?? new Date().toISOString(),
correlation_id: source.correlation_id,
});
return source;
}
async listSources(filter?: {
tenant_id?: string;
project_id?: string;
source_type?: SourceType;
status?: SourceStatus;
}): Promise<Source[]> {
return this.repo.list(filter as SourceListFilter | undefined);
}
}