fix(recovery): restore source removed by production documentation merge

This commit is contained in:
2026-11-10 03:42:10 +01:00
parent 585bdd4b44
commit 3d2fc6f913
957 changed files with 115754 additions and 0 deletions
@@ -0,0 +1,212 @@
/**
* src/adapters/document-intelligence-adapter.ts
* ================================================
* Tenant-agnostic Azure AI Document Intelligence client.
*
* Generalized from v0-datahub/lib/invoice-inbox/doc-intelligence.ts, which
* hardcoded the `prebuilt-invoice` model and a fixed invoice-shaped return
* type (amount_gross, vendor, vat, ...). Here `modelId` is a CALLER-SUPPLIED
* parameter — "prebuilt-invoice", "prebuilt-receipt", "prebuilt-layout",
* "prebuilt-read", or any custom-trained model id — and the adapter returns
* a generic field map instead of assuming any particular document shape.
*
* Evidence discipline (correctness-critical, not a style choice): a field is
* only ever populated with the value + confidence Document Intelligence
* actually reported. If DI did not return a value, the field comes back as
* { value: null, confidence: <DI's confidence if any, else null>, found: false }.
* Nothing here ever defaults, guesses, or backfills a value.
*/
export interface DocumentIntelligenceConfig {
endpoint: string;
apiKey: string;
/** Default: "2024-11-30" — the API version the production invoice pipeline used. */
apiVersion?: string;
/** Delay between result-polling attempts, ms. Default 2000. (Tests override to run fast.) */
pollIntervalMs?: number;
/** Delay before retrying after a 429, ms. Default 6000. (Tests override to run fast.) */
rateLimitRetryMs?: number;
/** Max result-polling attempts before timing out. Default 40. */
maxPollAttempts?: number;
/** Max analyze-request attempts (each 429 counts as one). Default 4. */
maxAnalyzeAttempts?: number;
}
export interface ExtractedField {
value: string | number | boolean | null;
confidence: number | null;
found: boolean;
}
export interface DocumentIntelligenceResult {
modelId: string;
apiVersion: string;
/** Full extracted text, when the API returns one (present across prebuilt models). */
content: string | null;
/** Top-level structured fields (documents[0].fields), normalized. Empty for pure layout/read. */
fields: Record<string, ExtractedField>;
/** Array-of-object fields (e.g. invoice/receipt line items), each item pre-normalized. */
arrayFields: Record<string, Array<Record<string, ExtractedField>>>;
/** documents[0].confidence, when the model reports a whole-document confidence. */
documentConfidence: number | null;
pageCount: number | null;
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/** Normalize one Document Intelligence field object into an evidence-based ExtractedField. */
function normalizeField(raw: unknown): ExtractedField {
const f = raw as Record<string, unknown> | null | undefined;
if (!f) return { value: null, confidence: null, found: false };
let value: string | number | boolean | null = null;
switch (f.type) {
case "string":
value = (f.valueString as string | undefined) ?? (f.content as string | undefined) ?? null;
break;
case "number":
value = (f.valueNumber as number | undefined) ?? null;
break;
case "integer":
value = (f.valueInteger as number | undefined) ?? null;
break;
case "date":
value = (f.valueDate as string | undefined) ?? null;
break;
case "time":
value = (f.valueTime as string | undefined) ?? null;
break;
case "phoneNumber":
value = (f.valuePhoneNumber as string | undefined) ?? null;
break;
case "countryRegion":
value = (f.valueCountryRegion as string | undefined) ?? null;
break;
case "selectionMark":
value = (f.valueSelectionMark as string | undefined) ?? null;
break;
case "currency": {
const cur = f.valueCurrency as Record<string, unknown> | undefined;
value = typeof cur?.amount === "number" ? (cur.amount as number) : null;
break;
}
case "boolean":
value = typeof f.valueBoolean === "boolean" ? (f.valueBoolean as boolean) : null;
break;
default:
value = (f.valueString as string | undefined) ?? (f.content as string | undefined) ?? null;
}
const found = value !== null && value !== undefined && value !== "";
const confidence = typeof f.confidence === "number" ? (f.confidence as number) : null;
return { value: found ? value : null, confidence, found };
}
function normalizeObjectFields(obj: Record<string, unknown> | null | undefined): Record<string, ExtractedField> {
const out: Record<string, ExtractedField> = {};
for (const [key, raw] of Object.entries(obj ?? {})) {
out[key] = normalizeField(raw);
}
return out;
}
export class DocumentIntelligenceAdapter {
constructor(private config: DocumentIntelligenceConfig) {}
static isConfigured(config: Partial<DocumentIntelligenceConfig>): boolean {
return !!(config.endpoint && config.apiKey);
}
/**
* Analyze a document buffer with the given model. Throws on any failure —
* callers must never interpret a caught error as "no fields found". A
* genuinely empty field is represented by { found: false } *inside a
* successful result*; a failed call must propagate as an error, not a
* fabricated empty success.
*/
async analyzeDocument(buf: Buffer, modelId: string): Promise<DocumentIntelligenceResult> {
const apiVersion = this.config.apiVersion || "2024-11-30";
const base = this.config.endpoint.replace(/\/$/, "");
const analyzeUrl = `${base}/documentintelligence/documentModels/${encodeURIComponent(modelId)}:analyze?api-version=${apiVersion}`;
const maxAnalyzeAttempts = this.config.maxAnalyzeAttempts ?? 4;
const rateLimitRetryMs = this.config.rateLimitRetryMs ?? 6000;
const pollIntervalMs = this.config.pollIntervalMs ?? 2000;
const maxPollAttempts = this.config.maxPollAttempts ?? 40;
let operationLocation = "";
for (let attempt = 0; attempt < maxAnalyzeAttempts; attempt++) {
const res = await fetch(analyzeUrl, {
method: "POST",
headers: { "Ocp-Apim-Subscription-Key": this.config.apiKey, "Content-Type": "application/json" },
body: JSON.stringify({ base64Source: buf.toString("base64") }),
});
if (res.status === 429) {
await sleep(rateLimitRetryMs);
continue;
}
if (res.status !== 202) {
const text = await res.text().catch(() => "");
throw new Error(`Document Intelligence analyze request failed: ${res.status} ${text.slice(0, 300)}`);
}
operationLocation = res.headers.get("operation-location") || "";
break;
}
if (!operationLocation) {
throw new Error(
"Document Intelligence analyze did not return an operation-location (rate-limited after retries, or misconfigured)",
);
}
let payload: Record<string, unknown> | null = null;
for (let i = 0; i < maxPollAttempts; i++) {
await sleep(pollIntervalMs);
const pollRes = await fetch(operationLocation, { headers: { "Ocp-Apim-Subscription-Key": this.config.apiKey } });
if (!pollRes.ok) {
const text = await pollRes.text().catch(() => "");
throw new Error(`Document Intelligence poll request failed: ${pollRes.status} ${text.slice(0, 300)}`);
}
const json = (await pollRes.json()) as Record<string, unknown>;
if (json.status === "succeeded") {
payload = json;
break;
}
if (json.status === "failed") {
throw new Error(`Document Intelligence analyze failed: ${JSON.stringify(json.error ?? json).slice(0, 500)}`);
}
}
if (!payload) {
throw new Error("Document Intelligence analyze timed out waiting for a result");
}
const analyzeResult = (payload.analyzeResult as Record<string, unknown>) ?? {};
const documents = analyzeResult.documents as Array<Record<string, unknown>> | undefined;
const doc = documents?.[0];
const rawFields = (doc?.fields as Record<string, unknown>) ?? {};
const fields: Record<string, ExtractedField> = {};
const arrayFields: Record<string, Array<Record<string, ExtractedField>>> = {};
for (const [key, raw] of Object.entries(rawFields)) {
const f = raw as Record<string, unknown>;
if (f?.type === "array" && Array.isArray(f.valueArray)) {
arrayFields[key] = (f.valueArray as Array<Record<string, unknown>>).map((item) =>
normalizeObjectFields(item?.valueObject as Record<string, unknown> | undefined),
);
} else {
fields[key] = normalizeField(f);
}
}
const pages = analyzeResult.pages as unknown[] | undefined;
return {
modelId,
apiVersion,
content: typeof analyzeResult.content === "string" ? (analyzeResult.content as string) : null,
fields,
arrayFields,
documentConfidence: typeof doc?.confidence === "number" ? (doc.confidence as number) : null,
pageCount: Array.isArray(pages) ? pages.length : null,
};
}
}
+254
View File
@@ -0,0 +1,254 @@
/**
* src/adapters/fabric-onelake-adapter.ts
* ========================================
* Tenant-agnostic Azure Fabric / OneLake SQL analytics endpoint adapter
* (adapter_id = "fabric_onelake_sql" on a 0pt Source).
*
* Generalized from two proven patterns in v0-datahub:
* - scripts/ops/cloud-bootstrap.mjs `aadToken()` — Entra service-principal
* OAuth2 client-credentials flow (POST login.microsoftonline.com/{tenant}
* /oauth2/v2.0/token, grant_type=client_credentials).
* - lib/warehouse/fabric-sql.ts `buildConfig()` / `runFabricQuery()` — using
* that token as an `azure-active-directory-access-token` credential on a
* `mssql` TDS connection to the Fabric SQL analytics endpoint, and
* returning `result.recordset` as plain rows.
*
* Both of those originals hardcode ONE customer's tenant/client/secret/
* endpoint as module-level constants or env vars (AZ_SP_*, FABRIC_SQL_ENDPOINT,
* kv-pts-datahub, ...). Nothing here is hardcoded: tenantId, clientId,
* clientSecret, sqlEndpoint and database are always caller-supplied, because
* this is a SIAX platform capability — any SIAX customer registers their own
* Fabric/OneLake warehouse as a Source and queries it through this adapter.
*
* Dependency note: `mssql` (the TDS driver used by fabric-sql.ts) is NOT a
* declared dependency of this repo yet, and package.json/pnpm-lock.yaml
* already have unrelated uncommitted changes from a separate in-flight
* session — so it is deliberately not added here to avoid colliding with
* that work. The driver is loaded lazily via a non-literal dynamic import
* (see `loadDriver`) so:
* - this file has zero hard dependency at import/bundle time (Next.js/
* webpack cannot statically resolve a non-literal specifier, so it does
* not fail the build even though "mssql" isn't installed), and
* - `tsc --noEmit` does not attempt to resolve module types for it either
* (a widened `string`-typed specifier makes `import(...)` type as
* `Promise<any>` instead of triggering module resolution).
* Once `mssql` is added to package.json (`pnpm add mssql`), query() works
* against a real endpoint with no code change here. Until then, `query()`
* throws a clear, actionable error — see docs/P0_QUERY_SOURCE_FLOW.md.
*
* SSRF guard: `connection_config.sqlEndpoint` is caller-supplied per-request
* (via POST /api/0pt/sources/{sourceId}/query -> source-query-service.ts)
* and, until now, only had its scheme stripped before being handed to the
* `mssql` driver as the outbound connection target — no hostname allowlist
* or private/loopback/link-local/cloud-metadata check. `getPool()` now
* resolves and validates the endpoint via
* `src/lib/security/ssrf-guard.ts::assertConnectionHostIsSafe` *before*
* constructing the pool, closing that gap defense-in-depth (this route
* already requires a Zitadel bearer token + tenant match — see
* src/http/tenant-guard.ts).
*/
import { assertConnectionHostIsSafe, type DnsLookupFn } from "../lib/security/ssrf-guard";
const DEFAULT_SQL_SCOPE = "https://database.windows.net/.default";
const TOKEN_REFRESH_SKEW_MS = 60_000;
const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
export interface FabricOneLakeCredentials {
/** Entra (Azure AD) tenant id the service-principal app registration lives in. */
tenantId: string;
/** Entra app (client) id — app-only, client-credentials grant. */
clientId: string;
/** Entra app client secret. Never logged, never persisted by this adapter. */
clientSecret: string;
}
export interface FabricOneLakeConnectionConfig {
credentials: FabricOneLakeCredentials;
/** Fabric SQL analytics endpoint host, e.g. "xxxx.datawarehouse.fabric.microsoft.com" (scheme optional). */
sqlEndpoint: string;
/** Target database / lakehouse SQL endpoint name, e.g. "lh_customer_datahub". */
database: string;
/** AAD scope used for the SQL token. Defaults to the Azure SQL / Fabric warehouse resource scope. */
scope?: string;
/** Per-request timeout passed to the mssql pool config. Default 30s. */
requestTimeoutMs?: number;
}
export interface FabricAccessToken {
accessToken: string;
expiresAtEpochMs: number;
}
export interface AdapterQueryResult<T = Record<string, unknown>> {
rows: T[];
rowCount: number;
durationMs: number;
}
/**
* OAuth2 client-credentials token acquisition for an Azure AD service
* principal — generalized 1:1 from v0-datahub's `aadToken()`. Exported
* standalone (not just as a private method) so it can be unit-tested and
* reused by callers that only need a token (e.g. to probe connectivity)
* without opening a SQL connection.
*/
export async function acquireFabricAccessToken(
credentials: FabricOneLakeCredentials,
scope: string = DEFAULT_SQL_SCOPE,
fetchImpl: typeof fetch = fetch,
): Promise<FabricAccessToken> {
const { tenantId, clientId, clientSecret } = credentials;
if (!tenantId || !clientId || !clientSecret) {
throw new Error("fabric_onelake_adapter: tenantId, clientId and clientSecret are all required");
}
const body = new URLSearchParams({
client_id: clientId,
client_secret: clientSecret,
grant_type: "client_credentials",
scope,
});
const res = await fetchImpl(`https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, {
method: "POST",
headers: { "content-type": "application/x-www-form-urlencoded" },
body,
});
const json: { access_token?: string; expires_in?: number; error?: string; error_description?: string } = await res.json();
if (!res.ok || !json.access_token) {
throw new Error(
`fabric_onelake_adapter: AAD token request failed (scope=${scope}): ${json.error ?? res.status} ${json.error_description ?? ""}`.trim(),
);
}
const expiresInMs = (typeof json.expires_in === "number" ? json.expires_in : 3600) * 1000;
return { accessToken: json.access_token, expiresAtEpochMs: Date.now() + expiresInMs };
}
function stripScheme(endpoint: string): string {
return endpoint.replace(/^https?:\/\//, "").replace(/\/+$/, "");
}
/** Minimal shape of the pieces of the `mssql` module this adapter actually uses. Kept as a structural type, never imported from the real package (see file header). */
interface MssqlLike {
ConnectionPool: new (config: Record<string, unknown>) => {
connect(): Promise<unknown>;
request(): {
input(name: string, value: unknown): unknown;
query(text: string): Promise<{ recordset?: unknown[] }>;
};
close(): Promise<void>;
};
}
/**
* Adapter for Source.adapter_id === "fabric_onelake_sql". One instance holds
* one registered Source's connection config; nothing is shared across
* tenants/customers, and no config here ever falls back to an env var.
*/
export class FabricOneLakeAdapter {
private token: FabricAccessToken | null = null;
private pool: InstanceType<MssqlLike["ConnectionPool"]> | null = null;
constructor(
private readonly config: FabricOneLakeConnectionConfig,
private readonly fetchImpl: typeof fetch = fetch,
/**
* Injectable DNS resolver for the SSRF guard (see `getPool()`).
* Overridable for tests, same rationale as `fetchImpl`; defaults to the
* real `node:dns` resolver in production (undefined -> ssrf-guard's own
* default).
*/
private readonly dnsLookupImpl?: DnsLookupFn,
) {}
private async getToken(): Promise<string> {
if (this.token && this.token.expiresAtEpochMs - TOKEN_REFRESH_SKEW_MS > Date.now()) {
return this.token.accessToken;
}
this.token = await acquireFabricAccessToken(this.config.credentials, this.config.scope, this.fetchImpl);
return this.token.accessToken;
}
/**
* Loads the `mssql` TDS driver at call time via a *non-literal* specifier.
* `moduleName` is deliberately widened to `string` (not the literal
* `"mssql"`) so TypeScript types the result as `Promise<any>` instead of
* resolving module types, and so bundlers cannot statically inline the
* specifier into a build-time-resolved chunk. Overridable for tests.
*/
private async loadDriver(): Promise<MssqlLike> {
const moduleName: string = "mssql";
try {
const mod = await import(moduleName);
return (mod.default ?? mod) as MssqlLike;
} catch (err) {
throw new Error(
`fabric_onelake_adapter: "mssql" package is not installed/resolvable. Add it to package.json ` +
`(dependencies.mssql) to enable live Fabric SQL query execution. Original error: ` +
`${err instanceof Error ? err.message : String(err)}`,
);
}
}
private async getPool(): Promise<InstanceType<MssqlLike["ConnectionPool"]>> {
if (this.pool) return this.pool;
// SSRF guard: resolve + validate the caller-supplied sqlEndpoint before
// doing anything else — in particular before loadDriver()/getToken(), so
// a blocked target never gets as far as a real outbound connection
// attempt (or spends an AAD token exchange on a request that will be
// refused anyway). Throws SsrfBlockedHostError for a private/loopback/
// link-local/cloud-metadata/unspecified address, or an unresolvable host.
await assertConnectionHostIsSafe(this.config.sqlEndpoint, this.dnsLookupImpl);
const sql = await this.loadDriver();
const token = await this.getToken();
const poolConfig = {
server: stripScheme(this.config.sqlEndpoint),
database: this.config.database,
options: { encrypt: true, trustServerCertificate: false },
authentication: {
type: "azure-active-directory-access-token",
options: { token },
},
pool: { max: 1, min: 0, idleTimeoutMillis: 30_000 },
requestTimeout: this.config.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
};
const pool = new sql.ConnectionPool(poolConfig);
await pool.connect();
this.pool = pool;
return pool;
}
/**
* Execute a query and return rows + timing. This adapter does NOT enforce
* row/time bounds itself — that is the job of the caller (see
* src/services/source-query-service.ts), which wraps this in a timeout and
* truncates the result for the synchronous path.
*/
async query<T = Record<string, unknown>>(queryText: string, params?: Record<string, unknown>): Promise<AdapterQueryResult<T>> {
const start = Date.now();
const pool = await this.getPool();
const request = pool.request();
if (params) {
for (const [key, value] of Object.entries(params)) {
request.input(key, value);
}
}
const result = await request.query(queryText);
const rows = (result.recordset ?? []) as T[];
return { rows, rowCount: rows.length, durationMs: Date.now() - start };
}
async close(): Promise<void> {
if (this.pool) {
await this.pool.close();
this.pool = null;
}
}
}
export function createFabricOneLakeAdapter(
config: FabricOneLakeConnectionConfig,
fetchImpl?: typeof fetch,
dnsLookupImpl?: DnsLookupFn,
): FabricOneLakeAdapter {
return new FabricOneLakeAdapter(config, fetchImpl, dnsLookupImpl);
}
+193
View File
@@ -0,0 +1,193 @@
/**
* src/adapters/graph-mailbox-adapter.ts
* ======================================
* Tenant-agnostic Microsoft Graph mailbox poller (app-only client-credentials).
*
* Generalized from v0-datahub/lib/invoice-inbox/poll.ts: there the mailbox
* (invoice@protosell.se), the Entra app, and the folder were all hardcoded
* module constants. Here every one of those is a CALLER-SUPPLIED parameter —
* any SIAX customer can register any mailbox (invoices, HR forms, contracts,
* receipts, ...) and poll it through this adapter. Nothing about invoices,
* vendors, or Protosell is baked in here — that logic (vendor-kb matching,
* dup-hash, DQ scoring, invoice classification) was intentionally left
* behind; it does not belong in a generic capture adapter.
*/
const GRAPH_BASE = "https://graph.microsoft.com/v1.0";
export interface GraphMailboxConfig {
/** Entra (Azure AD) tenant id the app registration lives in. */
tenantId: string;
/** Entra app (client) id — app-only, client-credentials grant. */
clientId: string;
/** Entra app client secret. Never logged, never persisted by this adapter. */
clientSecret: string;
/** Mailbox address to poll, e.g. "invoices@customer.com". */
mailboxAddress: string;
/** Mail folder to poll. Default: "inbox". */
folder?: string;
}
export interface GraphMessage {
id: string;
internetMessageId: string | null;
conversationId: string | null;
receivedAt: string | null;
senderName: string | null;
senderEmail: string | null;
toRecipients: string[];
subject: string | null;
bodyPreview: string | null;
/** HTML body stripped to plain text. Falls back to bodyPreview when body is absent. */
bodyText: string | null;
hasAttachments: boolean;
webLink: string | null;
}
export interface GraphAttachment {
id: string;
name: string | null;
contentType: string | null;
sizeBytes: number | null;
isPdf: boolean;
contentBytes: Buffer;
}
export interface PollMessagesOptions {
/** Only return messages received at/after this ISO timestamp (incremental polling). */
sinceISO?: string;
/** Page size per Graph request. Default 50. */
top?: number;
/** Max number of pages to walk. Default 2 (raise for backfills). */
maxPages?: number;
}
/** Strip HTML to plain text. Generic — no document-type-specific parsing. */
function stripHtml(html: string): string {
return (html || "")
.replace(/<style[\s\S]*?<\/style>/gi, " ")
.replace(/<script[\s\S]*?<\/script>/gi, " ")
.replace(/<[^>]+>/g, " ")
.replace(/&nbsp;/gi, " ")
.replace(/&amp;/gi, "&")
.replace(/&lt;/gi, "<")
.replace(/&gt;/gi, ">")
.replace(/\s+/g, " ")
.trim();
}
export class GraphMailboxAdapter {
private cachedToken: { token: string; expiresAt: number } | null = null;
constructor(private config: GraphMailboxConfig) {}
static isConfigured(config: Partial<GraphMailboxConfig>): boolean {
return !!(config.tenantId && config.clientId && config.clientSecret && config.mailboxAddress);
}
private async getToken(): Promise<string> {
if (this.cachedToken && this.cachedToken.expiresAt > Date.now() + 30_000) {
return this.cachedToken.token;
}
const { tenantId, clientId, clientSecret } = this.config;
if (!tenantId || !clientId || !clientSecret) {
throw new Error("GraphMailboxAdapter: missing tenantId/clientId/clientSecret");
}
const body = new URLSearchParams({
client_id: clientId,
client_secret: clientSecret,
grant_type: "client_credentials",
scope: "https://graph.microsoft.com/.default",
});
const res = await fetch(`https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, {
method: "POST",
headers: { "content-type": "application/x-www-form-urlencoded" },
body,
});
const json: Record<string, unknown> = await res.json();
if (!res.ok) {
throw new Error(`Graph token request failed: ${json.error || res.status} ${json.error_description || ""}`);
}
const expiresInMs = (typeof json.expires_in === "number" ? json.expires_in : 3600) * 1000;
this.cachedToken = { token: json.access_token as string, expiresAt: Date.now() + expiresInMs };
return this.cachedToken.token;
}
// Graph-API:t är genuint ostrukturerat JSON (value-array med fri form);
// full typning av hela Graph-ytan är egen insats — any här är medveten
// och avgränsad till denna adapter (eslint-undantag med skäl, DoD §13).
// eslint-disable-next-line @typescript-eslint/no-explicit-any
private async get(path: string): Promise<any> {
const token = await this.getToken();
const url = path.startsWith("http") ? path : `${GRAPH_BASE}${path}`;
const res = await fetch(url, { headers: { authorization: `Bearer ${token}` } });
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Graph-JSON är genuint ostrukturerat; full Graph-typning är egen insats
const json: any = await res.json();
if (!res.ok) {
throw new Error(`Graph GET ${path} -> ${json.error?.code || res.status}: ${json.error?.message || ""}`);
}
return json;
}
/** List messages in the configured mailbox/folder, newest first, optionally incremental. */
async pollMessages(opts: PollMessagesOptions = {}): Promise<GraphMessage[]> {
const folder = this.config.folder || "inbox";
const top = opts.top ?? 50;
const maxPages = opts.maxPages ?? 2;
const select =
"$select=id,internetMessageId,conversationId,receivedDateTime,from,toRecipients,subject,bodyPreview,hasAttachments,webLink,body";
let sinceFilter = "";
if (opts.sinceISO) {
sinceFilter = `&$filter=receivedDateTime ge ${new Date(opts.sinceISO).toISOString()}`;
}
let url =
`${GRAPH_BASE}/users/${encodeURIComponent(this.config.mailboxAddress)}/mailFolders/${encodeURIComponent(folder)}` +
`/messages?$top=${top}&$orderby=receivedDateTime desc&${select}${sinceFilter}`;
const out: GraphMessage[] = [];
for (let page = 0; page < maxPages && url; page++) {
const data = await this.get(url);
for (const m of data.value || []) {
out.push({
id: m.id,
internetMessageId: m.internetMessageId || null,
conversationId: m.conversationId || null,
receivedAt: m.receivedDateTime || null,
senderName: m.from?.emailAddress?.name || null,
senderEmail: m.from?.emailAddress?.address || null,
toRecipients: (m.toRecipients || []).map((r: { emailAddress?: { address?: string } }) => r.emailAddress?.address).filter(Boolean),
subject: m.subject || null,
bodyPreview: m.bodyPreview || null,
bodyText: m.body?.content ? stripHtml(m.body.content) : m.bodyPreview || null,
hasAttachments: !!m.hasAttachments,
webLink: m.webLink || null,
});
}
url = data["@odata.nextLink"] || "";
}
return out;
}
/** List and download every file attachment on a message. */
async listAttachments(messageId: string): Promise<GraphAttachment[]> {
const data = await this.get(
`/users/${encodeURIComponent(this.config.mailboxAddress)}/messages/${messageId}/attachments`,
);
const out: GraphAttachment[] = [];
for (const a of data.value || []) {
if (a["@odata.type"] !== "#microsoft.graph.fileAttachment" || !a.contentBytes) continue;
const contentType: string | null = a.contentType || null;
const name: string | null = a.name || null;
const isPdf = (contentType || "").includes("pdf") || /\.pdf$/i.test(name || "");
out.push({
id: a.id,
name,
contentType,
sizeBytes: typeof a.size === "number" ? a.size : null,
isPdf,
contentBytes: Buffer.from(a.contentBytes, "base64"),
});
}
return out;
}
}
+76
View File
@@ -0,0 +1,76 @@
/**
* src/adapters/local-blob-store.ts
* ==================================
* INTERIM local-disk blob store — NOT the real ST0RE bytes backend.
*
* KNOWN GAP: src/clients/st0re-client.ts only registers artifact METADATA
* (a pointer: provider/bucket/key/region). It has no endpoint that accepts
* and persists raw bytes — ST0RE does not yet expose a real object-storage
* write path for 0pt to call. Until it does, document-capture-service.ts
* and document-extraction-service.ts write attachment/extraction bytes here
* so the capability is functionally real end-to-end (download → persist →
* read back for extraction) instead of silently discarding the PDF bytes
* and only faking a metadata pointer.
*
* This is single-node, unencrypted-at-rest beyond the host filesystem, and
* has no retention/lifecycle management — do not treat it as production
* object storage. Swap it for a real ST0RE (or direct MinIO/Azure Blob)
* client the day one exists; every call site that constructs this class is
* marked "INTERIM" so the seam is easy to find.
*/
import { promises as fs } from "fs";
import path from "path";
import { isProductionRuntime } from "../platform/dependency-mode";
import { SiaxError } from "../platform/errors";
export interface LocalBlobWriteResult {
provider: "local_interim";
bucket: string;
key: string;
absolutePath: string;
sizeBytes: number;
}
export class LocalBlobStore {
constructor(private rootDir: string = process.env.OPT_LOCAL_BLOB_DIR || path.join(process.cwd(), ".data", "artifacts")) {}
/**
* A fallback that silently swaps durable object storage for a container's
* local filesystem is a double truth: `document-capture-service` registers
* an artifact pointer claiming the bytes are stored, while the bytes sit on
* an ephemeral single-node disk that is discarded on the next redeploy. In
* production the honest answer is DEPENDENCY_UNAVAILABLE (the capability is
* blocked because no ST0RE bytes path exists yet), never a fabricated
* success. Dev/test keep the adapter so the capture -> persist -> read-back
* flow stays exercisable end to end.
*
* Asserted per-operation rather than in the constructor so that merely
* WIRING the service (both services construct one eagerly as a default) is
* not itself fatal — only actually persisting or reading bytes is.
*/
private assertAllowed(operation: string): void {
if (isProductionRuntime()) {
throw new SiaxError(
"DEPENDENCY_UNAVAILABLE",
"0pt.local_blob_store",
`LocalBlobStore.${operation} is an interim dev/test adapter and may not persist document ` +
`bytes in a production runtime; no canonical ST0RE bytes path is configured`
);
}
}
async write(bucket: string, key: string, bytes: Buffer): Promise<LocalBlobWriteResult> {
this.assertAllowed("write");
const safeKey = key.replace(/\.\./g, "_");
const absolutePath = path.join(this.rootDir, bucket, safeKey);
await fs.mkdir(path.dirname(absolutePath), { recursive: true });
await fs.writeFile(absolutePath, bytes);
return { provider: "local_interim", bucket, key: safeKey, absolutePath, sizeBytes: bytes.length };
}
async read(bucket: string, key: string): Promise<Buffer> {
this.assertAllowed("read");
const absolutePath = path.join(this.rootDir, bucket, key);
return fs.readFile(absolutePath);
}
}
+279
View File
@@ -0,0 +1,279 @@
/**
* src/adapters/odoo-bills-adapter.ts
* =====================================
* `WritebackAdapter` (../adapters/writeback-adapter.ts) implementation for
* Odoo Vendor Bills (`account.move`, `move_type: 'in_invoice'`) --
* registered under `adapterId: "odoo_bills"` onto an
* `ExternalWritebackService` (../services/external-writeback-service.ts).
*
* PORTED FROM v0-datahub's `lib/odoo/invoice-bridge.ts` (read in full before
* writing this file), generalized as one WritebackAdapter implementation
* rather than a standalone module with its own Supabase-coupled pipeline
* function (`processInboxToOdoo`). What is a DIRECT PORT of proven logic,
* vs. what is NEW/GENERALIZED, stated precisely:
*
* DIRECT PORT (same behavior, same guard order, same reasoning):
* - `isReminder`/`duplicateFlag` guard -- invoice-bridge.ts's
* `processInboxToOdoo`: "if (invoice.is_reminder || invoice.duplicate_flag)
* return { ok:false, error: ... }" (a reminder or a detected duplicate is
* NEVER posted to Odoo). Ported verbatim as this adapter's first guard in
* `write()`.
* - "already posted, skip" guard -- invoice-bridge.ts: "if
* (invoice.odoo_move_id) { ...hoppar över...; return { ok:true, odoo_id:
* invoice.odoo_move_id } }". Ported as the `existingExternalId` check --
* this adapter's own idempotency defense-in-depth layer, complementing
* (not replacing) `ExternalWritebackService`'s own in-memory claim -- see
* that service's own module doc comment and writeback-adapter.ts's
* `WritebackAdapter.write` doc comment for why BOTH layers exist.
* - Partner matching by org-nr -- invoice-bridge.ts's
* `matchInvoiceToOdooPartner`: digits-only comparison against
* `odoo_export__res_partner.vat` so "556xxx-xxxx", "SE556xxxxxxxx01", and
* "5561234567" all match the same partner. Ported as
* `matchPartnerViaFabric` below, with ONE necessary substitution: the
* original queried via v0-datahub's own `@/lib/warehouse/client`
* (Next.js-app-local, not importable from this repo) -- this port uses
* 0pt's own already-built, equally generic
* `../adapters/fabric-onelake-adapter.ts` (built this same pass, for
* 0pt.document_ocr's own needs) against the SAME table name
* (`odoo_export__res_partner`) and the SAME digits-only `LIKE` query
* shape, parameterized instead of string-interpolated.
* - Vendor Bill request shape -- invoice-bridge.ts's `pushInvoiceToOdoo`:
* POSTs `{ model: "account.move", invoice: { partner_id, ref,
* invoice_date, invoice_date_due, invoice_line_ids: [...] } }` to
* `/api/odoo/invoices/write`. Ported verbatim as this adapter's own POST
* body -- same endpoint path convention, same field names, so an existing
* `/api/odoo/invoices/write` deployment needs no change to accept calls
* from this adapter.
* - Line synthesis fallback -- invoice-bridge.ts's `processInboxToOdoo`
* builds a single summary `invoice_line_ids` entry
* (`description: subject||vendor||"Faktura {number}"`, `quantity: 1,
* unit_price: amount_excl_vat`) when no itemized lines exist. Ported as
* `synthesizeSummaryLine` below, used only when `payload.lines` is absent.
*
* GENERALIZED / NEW (did not exist in the original, or could not be ported
* 1:1):
* - The Supabase read/claim/update-status statements in
* `processInboxToOdoo` (the `invoice_inbox` table CRUD, and its own
* atomic `UPDATE ... WHERE odoo_move_id IS NULL` claim) are NOT ported
* here -- that is `ExternalWritebackService`'s job now (its own
* adapter-agnostic in-memory claim, see that service's module doc
* comment), not this adapter's. This adapter is stateless per call.
* - `baseUrl`/`apiKey`/the Fabric credentials are all constructor
* parameters -- the original hardcoded `process.env.ODOO_BASE_URL`/
* `process.env.NEXT_PUBLIC_APP_URL`/etc as module-level env reads (v0-
* datahub is a single-tenant app; this repo is a SIAX platform capability
* any customer's tenant can register their own Odoo instance against, the
* same "nothing hardcoded, everything a caller-supplied credential"
* doctrine ../adapters/fabric-onelake-adapter.ts's own header already
* documents for the identical reason).
* - `fetchImpl`/`matchPartner` are injectable (default to the real `fetch`
* / `matchPartnerViaFabric`) purely so tests never make a real network
* call -- the original had no test coverage for this path at all
* (v0-datahub's invoice-bridge.ts has no accompanying test file); this
* port is genuinely tested (see tests/p0/odoo-bills-adapter.test.ts).
*/
import type { WritebackAdapter, WritebackAdapterContext, WritebackRequest, WritebackResult } from "./writeback-adapter";
import { FabricOneLakeAdapter, type FabricOneLakeConnectionConfig } from "./fabric-onelake-adapter";
/** One itemized Vendor Bill line -- same shape as invoice-bridge.ts's `InvoiceForOdoo.lines`. */
export interface OdooBillLine {
description: string;
quantity: number;
unit_price: number;
vat_rate?: number;
account_bas?: number;
}
/**
* The structured record this adapter expects as `WritebackRequest.payload`
* -- same field set as invoice-bridge.ts's `InvoiceForOdoo`, plus the guard
* fields (`is_reminder`/`duplicate_flag`/`existing_external_id`) that
* `processInboxToOdoo` read directly off its own `invoice_inbox` row (this
* adapter has no row of its own to read them from, so the caller supplies
* them on the payload instead).
*/
export interface OdooBillPayload {
vendor_name: string;
vendor_org_nr?: string;
/** Pre-resolved Odoo res.partner id -- when absent and `vendor_org_nr` is set, this adapter
* attempts to resolve one via partner matching (see this file's module doc comment). */
partner_id?: number;
invoice_number: string;
ocr?: string;
amount_gross: number;
amount_excl_vat?: number;
vat?: number;
currency: string;
invoice_date: string;
due_date?: string;
description?: string;
lines?: readonly OdooBillLine[];
/** Mirrors invoice_inbox.is_reminder -- a payment reminder must never be posted as a new Bill. */
is_reminder?: boolean;
/** Mirrors invoice_inbox.duplicate_flag -- a detected duplicate must never be posted. */
duplicate_flag?: boolean;
/** When the caller already knows this record was posted (e.g. re-delivering a webhook), pass
* the known Odoo id here -- this adapter treats it as already-done and returns it unchanged,
* exactly mirroring invoice-bridge.ts's own `invoice.odoo_move_id` early-return. */
existing_external_id?: string;
}
export type OdooPartnerMatcher = (
orgNr: string,
ctx: WritebackAdapterContext,
) => Promise<{ partner_id?: number; partner_name?: string }>;
export interface OdooBillsAdapterConfig {
/** Base URL of the deployment exposing `/api/odoo/invoices/write` (same route shape as
* v0-datahub's own `/api/odoo/invoices/write` this was ported from). */
baseUrl: string;
/** Bearer credential for that route. Optional ONLY when `allowUnauthenticated` is explicitly
* set (e.g. a route that authenticates by network policy instead) -- see `isConfigured`. */
apiKey?: string;
allowUnauthenticated?: boolean;
/** Fabric/OneLake connection used for org-nr -> Odoo partner matching (this file's own
* `matchPartnerViaFabric`), when `matchPartner` is not explicitly supplied. Optional --
* without it (and without an explicit `matchPartner`), partner matching is skipped entirely
* and the Bill is posted without a partner link, mirroring invoice-bridge.ts's own
* `matchInvoiceToOdooPartner` catch-all ("Ingen partner match — skickar utan
* partner-koppling"). */
fabric?: FabricOneLakeConnectionConfig;
/** Table name for the partner lookup query -- defaults to `odoo_export__res_partner`, the
* exact table invoice-bridge.ts's own `matchInvoiceToOdooPartner` queried. */
partnerTable?: string;
/** Injectable partner matcher, for tests or a caller with its own partner-resolution source.
* When supplied, this takes priority over `fabric`. */
matchPartner?: OdooPartnerMatcher;
/** Injectable `fetch`, so tests never make a real network call. */
fetchImpl?: typeof fetch;
}
/**
* Digits-only org-nr -> Odoo partner match, ported 1:1 from invoice-bridge.ts's
* `matchInvoiceToOdooPartner` (see this file's module doc comment). The only
* substitution is the query transport: `FabricOneLakeAdapter.query` instead
* of v0-datahub's `@/lib/warehouse/client` -- same table, same digits-only
* `LIKE` comparison so "556xxx-xxxx"/"SE556xxxxxxxx01"/"5561234567" all
* resolve to the same partner, now parameterized instead of string-interpolated.
*/
export async function matchPartnerViaFabric(
fabric: FabricOneLakeAdapter,
orgNr: string,
partnerTable: string = "odoo_export__res_partner",
): Promise<{ partner_id?: number; partner_name?: string }> {
const digits = orgNr.replace(/[^0-9]/g, "");
if (!digits) return {};
try {
const result = await fabric.query<{ id: number; name: string; vat: string }>(
`SELECT TOP 1 id, name, vat FROM ${partnerTable} WHERE REPLACE(REPLACE(vat,'-',''),' ','') LIKE '%' + @digits + '%'`,
{ digits },
);
if (result.rows.length > 0) {
return { partner_id: result.rows[0]!.id, partner_name: result.rows[0]!.name };
}
return {};
} catch {
// Same catch-all as invoice-bridge.ts's own matchInvoiceToOdooPartner: an unreachable/
// misconfigured warehouse must never fail the whole write -- it degrades to "post without
// a partner link", not a thrown error.
return {};
}
}
/** Builds the single synthesized `invoice_line_ids` entry used when `payload.lines` is absent
* -- ported verbatim from invoice-bridge.ts's `processInboxToOdoo` line-synthesis block. */
function synthesizeSummaryLine(payload: OdooBillPayload): OdooBillLine {
const exclVat = payload.amount_excl_vat ?? payload.amount_gross;
const description = (payload.description || payload.vendor_name || `Faktura ${payload.invoice_number}`).trim();
return { description, quantity: 1, unit_price: exclVat };
}
function isOdooBillPayload(payload: Readonly<Record<string, unknown>>): payload is OdooBillPayload & Record<string, unknown> {
return (
typeof payload.vendor_name === "string" &&
typeof payload.invoice_number === "string" &&
typeof payload.amount_gross === "number" &&
typeof payload.currency === "string" &&
typeof payload.invoice_date === "string"
);
}
export class OdooBillsAdapter implements WritebackAdapter {
readonly adapterId = "odoo_bills";
private fabricAdapter: FabricOneLakeAdapter | null;
constructor(private config: OdooBillsAdapterConfig) {
this.fabricAdapter = config.fabric ? new FabricOneLakeAdapter(config.fabric, config.fetchImpl) : null;
}
isConfigured(): boolean {
return !!this.config.baseUrl && (!!this.config.apiKey || this.config.allowUnauthenticated === true);
}
async write(request: WritebackRequest, ctx: WritebackAdapterContext): Promise<WritebackResult> {
if (!this.isConfigured()) {
return { ok: false, error: "odoo_not_configured" };
}
if (!isOdooBillPayload(request.payload)) {
return { ok: false, error: "invalid_payload_missing_required_fields" };
}
const payload = request.payload as unknown as OdooBillPayload;
// Guard 1: reminders/duplicates are never posted -- ported verbatim from
// invoice-bridge.ts's processInboxToOdoo (see this file's module doc comment).
if (payload.is_reminder) return { ok: false, error: "is_reminder" };
if (payload.duplicate_flag) return { ok: false, error: "duplicate" };
// Guard 2: already posted -- idempotent no-op, ported verbatim.
if (payload.existing_external_id) {
return { ok: true, externalId: payload.existing_external_id };
}
// Partner match (best-effort, never fails the write) -- ported verbatim in spirit.
let partnerId = payload.partner_id;
if (!partnerId && payload.vendor_org_nr) {
if (this.config.matchPartner) {
const matched = await this.config.matchPartner(payload.vendor_org_nr, ctx);
partnerId = matched.partner_id;
} else if (this.fabricAdapter) {
const matched = await matchPartnerViaFabric(this.fabricAdapter, payload.vendor_org_nr, this.config.partnerTable);
partnerId = matched.partner_id;
}
}
const lines = payload.lines && payload.lines.length > 0 ? payload.lines : [synthesizeSummaryLine(payload)];
const fetchImpl = this.config.fetchImpl ?? fetch;
let res: Response;
try {
res = await fetchImpl(`${this.config.baseUrl}/api/odoo/invoices/write`, {
method: "POST",
headers: {
"Content-Type": "application/json",
...(this.config.apiKey ? { Authorization: `Bearer ${this.config.apiKey}` } : {}),
},
body: JSON.stringify({
model: "account.move",
invoice: {
partner_id: partnerId,
ref: payload.invoice_number,
invoice_date: payload.invoice_date,
invoice_date_due: payload.due_date,
invoice_line_ids: lines.map((l) => ({ name: l.description, quantity: l.quantity, price_unit: l.unit_price })),
},
}),
});
} catch (e) {
return { ok: false, error: `odoo_network_error: ${e instanceof Error ? e.message : String(e)}` };
}
if (!res.ok) {
return { ok: false, error: `odoo_http_${res.status}` };
}
const result: { ok?: boolean; odoo_id?: number; error?: string } = await res.json();
if (!result.ok || !result.odoo_id) {
return { ok: false, error: result.error ?? "odoo_write_failed" };
}
return { ok: true, externalId: String(result.odoo_id) };
}
}
+71
View File
@@ -0,0 +1,71 @@
/**
* src/adapters/writeback-adapter.ts
* ===================================
* The adapter interface `ExternalWritebackService` (../services/
* external-writeback-service.ts) writes through. One implementation per
* external system (e.g. odoo-bills-adapter.ts's OdooBillsAdapter for Odoo
* Vendor Bills) -- the service itself has zero knowledge of Odoo, Salesforce,
* or any other concrete external system; it only knows "an adapter with this
* adapterId exists, is configured, and can write a record."
*/
export interface WritebackAdapterContext {
tenant_id: string;
project_id: string;
actor_id: string;
correlation_id: string;
}
export interface WritebackRequest {
recordType: string;
payload: Readonly<Record<string, unknown>>;
/** Same idempotency key the caller gave ExternalWritebackService -- forwarded so an adapter
* with its own persistent storage (unlike the generic service's in-memory claim) can ALSO
* claim defensively at its own layer. See WritebackAdapter.write's own doc comment. */
idempotencyKey: string;
}
export interface WritebackResult {
ok: boolean;
/** The external system's own id for the record that was created (or already existed --
* see WritebackAdapter.write's own doc comment on idempotent re-calls), when ok is true. */
externalId?: string;
/** A short, stable, machine-readable reason code when ok is false (e.g. "is_reminder",
* "duplicate", "odoo_http_500") -- NOT a free-form human sentence, so callers/tests can
* assert on it reliably. */
error?: string;
}
/**
* One external-system write adapter. Registered onto an
* `ExternalWritebackService` via `registerAdapter` under a stable `adapterId`
* (e.g. "odoo_bills") that `ExternalWriteRequest.adapter_id` references.
*/
export interface WritebackAdapter {
readonly adapterId: string;
/** Whether this adapter has everything it needs (base URL, credentials, etc) to actually
* write -- mirrors DocumentIntelligenceAdapter.isConfigured's "refuse to fabricate a result"
* convention (../adapters/document-intelligence-adapter.ts): an unconfigured adapter must
* never be silently treated as a no-op success. */
isConfigured(): boolean;
/**
* Writes one record to the external system.
*
* IDEMPOTENCY, TWO LAYERS (per this task's own instruction not to regress the proven
* idempotent-claim logic ported from v0-datahub's invoice-bridge.ts):
* 1. `ExternalWritebackService` (the caller) already claims `idempotencyKey` in its own
* in-memory map BEFORE ever calling this method, so under normal operation `write` is
* only ever called once per logical record, from one service instance.
* 2. That in-memory claim is process-local and does not survive a restart, nor protect a
* multi-instance deployment. An adapter backed by its own persistent storage (like
* OdooBillsAdapter, when a caller supplies an already-posted record's
* `existing_external_id` in the payload) MUST ALSO refuse to re-post a record that
* already carries an external id -- exactly the "Redan bokförd i Odoo... hoppar över"
* early-return invoice-bridge.ts's own `processInboxToOdoo` performs. This is the
* genuine defense-in-depth the original proven code had; a `WritebackAdapter`
* implementation with real persistent storage should follow the same doctrine.
*/
write(request: WritebackRequest, ctx: WritebackAdapterContext): Promise<WritebackResult>;
}