Files
0pt/app/api/0pt/documents/structure/route.ts
T
admin dade37ee05
CI (SIAX Cloud) / sonar (pull_request) Skipped
masterplan-lock / check (pull_request) Successful in 4s
masterplan-lock / check (push) Successful in 5s
CI (SIAX Cloud) / security (pull_request) Successful in 16s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 16s
CI (SIAX Cloud) / integration (pull_request) Successful in 9m58s
CI (SIAX Cloud) / quality (pull_request) Successful in 10m25s
feat(credits): one-time trial grant so the ledger runs in production
- Ledger accepts kind='grant' (a server-owned balance grant, not a payment).
- OPT_TRIAL_CREDITS=N grants N credits once per tenant (idempotent key
  trial:<tenant>) on balance/structure, so reserve -> settle is exercised
  before any customer purchase.
- sql/020 extends the kind constraint (re-runnable).
- Tests for the grant helper; full suite 1345 passed.
2026-11-02 22:05:06 +01:00

455 lines
15 KiB
TypeScript

import { NextResponse, type NextRequest } from 'next/server';
import { Doc0SourceClient, Doc0SourceUnavailableError } from '@/src/document-workbench/doc0-source-client';
import { Inf0OutcomeUnknownError } from '@/src/document-workbench/inf0-gateway-client';
import { createInf0Gateway } from '@/src/document-workbench/inf0-gateway';
import { PostgresCreditLedger } from '@/src/document-workbench/credit-ledger';
import { ensureTrialCredits, readTrialCredits } from '@/src/document-workbench/trial-credits';
import { creditsForUsage, readCreditPriceBook } from '@/src/document-workbench/price-book';
import { structureDocument } from '@/src/document-workbench/structure-document';
import { exportValidatedOutput } from '@/src/document-workbench/export-validated-output';
import type {
OutputFieldContract,
OutputFieldType,
} from '@/src/document-workbench/structured-output-validation';
import { readDocumentWorkbenchRuntime } from '@/src/document-workbench/runtime-config';
import { createProjectAuthorizer } from '@/src/document-workbench/project-access';
import { getRepositorySql, repositoryDatabaseUrl } from '@/src/repositories/db';
import { isValidScopeId, resolveVerifiedScope } from '@/src/http/verified-scope';
export const runtime = 'nodejs';
export const dynamic = 'force-dynamic';
const FIELD_TYPES: readonly OutputFieldType[] = ['string', 'integer', 'boolean', 'decimal', 'date'];
const DATA_CLASSES = ['none', 'low', 'medium', 'high', 'restricted'] as const;
function csv(value: string | undefined): string[] {
return (value ?? '')
.split(',')
.map((part) => part.trim())
.filter(Boolean);
}
function modelPolicy() {
const allowedModelIds = csv(process.env.OPT_ALLOWED_MODELS);
const allowedDataClasses = csv(process.env.OPT_ALLOWED_DATA_CLASSES);
const destination = process.env.OPT_MODEL_DESTINATION;
const maxInputTokens = Number(process.env.OPT_MODEL_MAX_INPUT_TOKENS ?? '0');
const maxOutputTokens = Number(process.env.OPT_MODEL_MAX_OUTPUT_TOKENS ?? '0');
if (
allowedModelIds.length === 0 ||
allowedDataClasses.length === 0 ||
(destination !== 'internal' && destination !== 'external') ||
!Number.isSafeInteger(maxInputTokens) ||
maxInputTokens <= 0 ||
!Number.isSafeInteger(maxOutputTokens) ||
maxOutputTokens <= 0
) {
return undefined;
}
return {
allowedModelIds,
allowedDataClasses,
destination: destination as 'internal' | 'external',
externalTransferAllowed: process.env.OPT_EXTERNAL_TRANSFER_ALLOWED === 'true',
maxInputTokens,
maxOutputTokens,
};
}
function parseFields(raw: unknown): OutputFieldContract[] | undefined {
if (!Array.isArray(raw) || raw.length === 0 || raw.length > 100) return undefined;
const fields: OutputFieldContract[] = [];
const seen = new Set<string>();
for (const item of raw) {
if (!item || typeof item !== 'object' || Array.isArray(item)) return undefined;
const field = item as Record<string, unknown>;
if (
typeof field.name !== 'string' ||
!/^[A-Za-z_][A-Za-z0-9_]{0,63}$/.test(field.name) ||
seen.has(field.name) ||
typeof field.type !== 'string' ||
!FIELD_TYPES.includes(field.type as OutputFieldType)
) {
return undefined;
}
seen.add(field.name);
fields.push({ name: field.name, type: field.type as OutputFieldType, required: true });
}
return fields;
}
interface StructureBody {
project_id?: unknown;
document_id?: unknown;
version_id?: unknown;
checksum_sha256?: unknown;
pipeline_stage?: unknown;
instruction?: unknown;
fields?: unknown;
model_id?: unknown;
data_class?: unknown;
format?: unknown;
}
/**
* POST /api/0pt/documents/structure
*
* Upload → DOC0 segments → INF0 model → mechanically validated, evidence-bound
* output (optionally exported). Credits are reserved before the model runs and
* settled to the measured usage afterwards.
*
* Nothing is trusted from the client except the requested target fields and
* instruction. Source text is re-read from DOC0 and every returned value must
* point at an exact span in that text.
*/
export async function POST(request: NextRequest) {
const verified = resolveVerifiedScope(request);
if ('response' in verified) return verified.response;
const { tenantId, actorId } = verified.scope;
const runtimeConfig = readDocumentWorkbenchRuntime();
const gateway = createInf0Gateway();
if (!runtimeConfig.doc0 || !gateway) {
return NextResponse.json(
{
error: 'WORKBENCH_NOT_CONFIGURED',
message: 'Document structuring is not available until DOC0 and INF0 are configured',
},
{ status: 503 },
);
}
const policy = modelPolicy();
if (!policy) {
return NextResponse.json(
{
error: 'MODEL_POLICY_NOT_CONFIGURED',
message: 'Document structuring is not available until the model policy is configured',
},
{ status: 503 },
);
}
// Explicit pilot switch: `OPT_CREDITS_MODE=off` runs structuring without a
// wallet. It is deliberately NOT a silent fallback — the response reports
// `credits: null` and the operator has named the mode.
const creditsEnabled =
process.env.OPT_CREDITS_MODE !== 'off' &&
Boolean(readCreditPriceBook()) &&
Boolean(repositoryDatabaseUrl());
const priceBook = creditsEnabled ? readCreditPriceBook() : undefined;
const databaseUrl = creditsEnabled ? repositoryDatabaseUrl() : undefined;
if (!creditsEnabled && process.env.OPT_CREDITS_MODE !== 'off') {
return NextResponse.json(
{
error: 'CREDITS_NOT_CONFIGURED',
message: 'Document structuring is not available until credits are configured',
},
{ status: 503 },
);
}
let body: StructureBody;
try {
body = (await request.json()) as StructureBody;
} catch {
return NextResponse.json({ error: 'INVALID_BODY', message: 'JSON body required' }, { status: 400 });
}
const projectId = body.project_id;
const documentId = body.document_id;
const versionId = body.version_id;
const checksum = body.checksum_sha256;
const pipelineStage = body.pipeline_stage;
const instruction = body.instruction;
const modelId = body.model_id;
const dataClass = body.data_class ?? 'low';
const format = body.format ?? 'json';
if (pipelineStage !== 'parsed') {
return NextResponse.json(
{
error: 'DOCUMENT_NOT_PARSED',
message:
'The document has no verified text layer yet (parse_failed or not parsed). OCR is required before structuring.',
},
{ status: 422 },
);
}
if (
!isValidScopeId(projectId) ||
!isValidScopeId(documentId) ||
!isValidScopeId(versionId) ||
typeof checksum !== 'string' ||
!/^[0-9a-f]{64}$/.test(checksum) ||
typeof instruction !== 'string' ||
!instruction.trim() ||
instruction.length > 4000 ||
typeof modelId !== 'string' ||
!policy.allowedModelIds.includes(modelId) ||
typeof dataClass !== 'string' ||
!(DATA_CLASSES as readonly string[]).includes(dataClass) ||
(format !== 'json' && format !== 'csv')
) {
return NextResponse.json(
{ error: 'INVALID_REQUEST', message: 'Invalid document structure request' },
{ status: 400 },
);
}
const fields = parseFields(body.fields);
if (!fields) {
return NextResponse.json(
{ error: 'INVALID_FIELDS', message: 'A valid field contract is required' },
{ status: 400 },
);
}
const authorize = createProjectAuthorizer();
if (!(await authorize({ tenantId, projectId, actorId }))) {
return NextResponse.json(
{ error: 'OPT_ACCESS_DENIED', message: 'Project access is not authorized' },
{ status: 403 },
);
}
const correlationId = request.headers.get('x-request-id') ?? crypto.randomUUID();
const context = {
tenantId,
projectId,
actorId,
capability: '0pt.document.structure.v1',
correlationId,
idempotencyKey: correlationId,
};
let sources;
try {
sources = await new Doc0SourceClient({
baseUrl: runtimeConfig.doc0.baseUrl,
serviceToken: runtimeConfig.doc0.serviceToken,
}).read({
tenantId,
projectId,
actorId: runtimeConfig.doc0.actorId ?? actorId,
requestId: correlationId,
receipt: {
documentId,
versionId,
checksumSha256: checksum,
pipelineStage: 'parsed',
},
});
} catch (error: unknown) {
if (error instanceof Doc0SourceUnavailableError) {
return NextResponse.json(
{
error: 'DOC0_SOURCE_UNAVAILABLE',
message:
'The extracted source could not be verified (version, checksum, page cap or tenant). Review or reconciliation is required.',
},
{ status: 422 },
);
}
return NextResponse.json(
{ error: 'DOC0_SOURCE_UNAVAILABLE', message: 'The extracted source is unavailable' },
{ status: 502 },
);
}
console.log('[0pt-workbench] doc0 source read:', sources.sources.length, 'segments');
const sourceChars = sources.sources.reduce((total, source) => total + source.text.length, 0);
const estimatedInputTokens = Math.ceil(JSON.stringify(sources.sources).length / 4) + 1024;
if (estimatedInputTokens > policy.maxInputTokens) {
return NextResponse.json(
{
error: 'SOURCE_TOO_LARGE',
message: 'The document is larger than the configured model input limit',
},
{ status: 413 },
);
}
const maximumCredits =
creditsEnabled && priceBook
? creditsForUsage(priceBook, {
pages: Math.max(1, Math.ceil(sourceChars / 1800)),
inputTokens: estimatedInputTokens,
outputTokens: policy.maxOutputTokens,
})
: 0;
const reservationId = crypto.randomUUID();
const ledger =
creditsEnabled && databaseUrl ? new PostgresCreditLedger(getRepositorySql(databaseUrl)) : undefined;
if (ledger) {
try {
await ensureTrialCredits(ledger, tenantId, readTrialCredits());
await ledger.apply({
tenantId,
idempotencyKey: `reserve:${reservationId}`,
kind: 'reserve',
reference: reservationId,
amount: maximumCredits,
});
} catch (error: unknown) {
const message = error instanceof Error ? error.message : 'Credit reservation failed';
if (/Insufficient available credits/i.test(message)) {
return NextResponse.json(
{
error: 'INSUFFICIENT_CREDITS',
message: 'Not enough credits for this job',
required_credits: maximumCredits,
},
{ status: 402 },
);
}
return NextResponse.json({ error: 'CREDITS_UNAVAILABLE', message }, { status: 503 });
}
}
try {
const result = await structureDocument({
gateway,
request: {
context,
admission: {
tenantId,
projectId,
jobId: `doc:${documentId}:${correlationId}`,
modelId,
dataClass,
maxInputTokens: policy.maxInputTokens,
maxOutputTokens: policy.maxOutputTokens,
maximumCredits,
},
policy: {
tenantId,
projectId,
allowedModelIds: policy.allowedModelIds,
allowedDataClasses: policy.allowedDataClasses,
destination: policy.destination,
externalTransferAllowed: policy.externalTransferAllowed,
maxInputTokens: policy.maxInputTokens,
maxOutputTokens: policy.maxOutputTokens,
},
reservation: ledger
? {
tenantId,
projectId,
jobId: `doc:${documentId}:${correlationId}`,
status: 'reserved',
credits: maximumCredits,
}
: undefined,
},
countTokens: async (messages) => Math.ceil(JSON.stringify(messages).length / 4),
instructions: instruction,
fields,
sources: sources.sources,
});
const charged =
ledger && priceBook
? Math.min(
maximumCredits,
creditsForUsage(priceBook, {
pages: Math.max(1, Math.ceil(sourceChars / 1800)),
inputTokens: result.model.promptTokens,
outputTokens: result.model.completionTokens,
}),
)
: 0;
let balance;
if (ledger) {
try {
balance = (
await ledger.apply({
tenantId,
idempotencyKey: `settle:${reservationId}`,
kind: 'settle',
reference: reservationId,
amount: charged,
})
).balance;
} catch {
// The model result is real; keep the reservation for reconciliation
// rather than reporting a balance we could not write.
balance = undefined;
}
}
if (result.status !== 'mechanically_validated') {
return NextResponse.json(
{
status: 'needs_review',
issues: result.issues,
model: result.model,
credits: ledger ? { reserved: maximumCredits, charged, balance } : null,
},
{ status: 200 },
);
}
const exported =
format === 'json'
? exportValidatedOutput({
format: 'json',
fields,
sources: sources.sources,
value: result.value,
evidence: result.evidence,
})
: exportValidatedOutput({
format: 'csv',
fields,
sources: sources.sources,
value: result.value,
evidence: result.evidence,
});
return NextResponse.json(
{
status: 'mechanically_validated',
value: result.value,
evidence: result.evidence,
model: result.model,
credits: ledger ? { reserved: maximumCredits, charged, balance } : null,
export:
exported.status === 'ready'
? { media_type: exported.mediaType, content: exported.content }
: undefined,
},
{ status: 200 },
);
} catch (error: unknown) {
if (error instanceof Inf0OutcomeUnknownError) {
// Never release a reservation whose model call may have completed.
return NextResponse.json(
{
status: 'outcome_unknown',
message:
'The model call may have completed. The credit reservation is kept until its usage is reconciled.',
reservation_id: reservationId,
},
{ status: 202 },
);
}
if (ledger) {
try {
await ledger.apply({
tenantId,
idempotencyKey: `release:${reservationId}`,
kind: 'release',
reference: reservationId,
amount: 0,
});
} catch {
/* the reservation stays for reconciliation */
}
}
const message = error instanceof Error ? error.message : 'Structuring failed';
console.error('[0pt-workbench] structure failed:', message);
return NextResponse.json(
{ error: 'STRUCTURE_FAILED', message, reservation_id: reservationId },
{ status: 502 },
);
}
}