CI (SIAX Cloud) / sonar (pull_request) Skipped
masterplan-lock / check (pull_request) Successful in 4s
masterplan-lock / check (push) Successful in 5s
CI (SIAX Cloud) / security (pull_request) Successful in 16s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 16s
CI (SIAX Cloud) / integration (pull_request) Successful in 9m58s
CI (SIAX Cloud) / quality (pull_request) Successful in 10m25s
- Ledger accepts kind='grant' (a server-owned balance grant, not a payment). - OPT_TRIAL_CREDITS=N grants N credits once per tenant (idempotent key trial:<tenant>) on balance/structure, so reserve -> settle is exercised before any customer purchase. - sql/020 extends the kind constraint (re-runnable). - Tests for the grant helper; full suite 1345 passed.
455 lines
15 KiB
TypeScript
455 lines
15 KiB
TypeScript
import { NextResponse, type NextRequest } from 'next/server';
|
|
import { Doc0SourceClient, Doc0SourceUnavailableError } from '@/src/document-workbench/doc0-source-client';
|
|
import { Inf0OutcomeUnknownError } from '@/src/document-workbench/inf0-gateway-client';
|
|
import { createInf0Gateway } from '@/src/document-workbench/inf0-gateway';
|
|
import { PostgresCreditLedger } from '@/src/document-workbench/credit-ledger';
|
|
import { ensureTrialCredits, readTrialCredits } from '@/src/document-workbench/trial-credits';
|
|
import { creditsForUsage, readCreditPriceBook } from '@/src/document-workbench/price-book';
|
|
import { structureDocument } from '@/src/document-workbench/structure-document';
|
|
import { exportValidatedOutput } from '@/src/document-workbench/export-validated-output';
|
|
import type {
|
|
OutputFieldContract,
|
|
OutputFieldType,
|
|
} from '@/src/document-workbench/structured-output-validation';
|
|
import { readDocumentWorkbenchRuntime } from '@/src/document-workbench/runtime-config';
|
|
import { createProjectAuthorizer } from '@/src/document-workbench/project-access';
|
|
import { getRepositorySql, repositoryDatabaseUrl } from '@/src/repositories/db';
|
|
import { isValidScopeId, resolveVerifiedScope } from '@/src/http/verified-scope';
|
|
|
|
export const runtime = 'nodejs';
|
|
export const dynamic = 'force-dynamic';
|
|
|
|
const FIELD_TYPES: readonly OutputFieldType[] = ['string', 'integer', 'boolean', 'decimal', 'date'];
|
|
const DATA_CLASSES = ['none', 'low', 'medium', 'high', 'restricted'] as const;
|
|
|
|
function csv(value: string | undefined): string[] {
|
|
return (value ?? '')
|
|
.split(',')
|
|
.map((part) => part.trim())
|
|
.filter(Boolean);
|
|
}
|
|
|
|
function modelPolicy() {
|
|
const allowedModelIds = csv(process.env.OPT_ALLOWED_MODELS);
|
|
const allowedDataClasses = csv(process.env.OPT_ALLOWED_DATA_CLASSES);
|
|
const destination = process.env.OPT_MODEL_DESTINATION;
|
|
const maxInputTokens = Number(process.env.OPT_MODEL_MAX_INPUT_TOKENS ?? '0');
|
|
const maxOutputTokens = Number(process.env.OPT_MODEL_MAX_OUTPUT_TOKENS ?? '0');
|
|
if (
|
|
allowedModelIds.length === 0 ||
|
|
allowedDataClasses.length === 0 ||
|
|
(destination !== 'internal' && destination !== 'external') ||
|
|
!Number.isSafeInteger(maxInputTokens) ||
|
|
maxInputTokens <= 0 ||
|
|
!Number.isSafeInteger(maxOutputTokens) ||
|
|
maxOutputTokens <= 0
|
|
) {
|
|
return undefined;
|
|
}
|
|
return {
|
|
allowedModelIds,
|
|
allowedDataClasses,
|
|
destination: destination as 'internal' | 'external',
|
|
externalTransferAllowed: process.env.OPT_EXTERNAL_TRANSFER_ALLOWED === 'true',
|
|
maxInputTokens,
|
|
maxOutputTokens,
|
|
};
|
|
}
|
|
|
|
function parseFields(raw: unknown): OutputFieldContract[] | undefined {
|
|
if (!Array.isArray(raw) || raw.length === 0 || raw.length > 100) return undefined;
|
|
const fields: OutputFieldContract[] = [];
|
|
const seen = new Set<string>();
|
|
for (const item of raw) {
|
|
if (!item || typeof item !== 'object' || Array.isArray(item)) return undefined;
|
|
const field = item as Record<string, unknown>;
|
|
if (
|
|
typeof field.name !== 'string' ||
|
|
!/^[A-Za-z_][A-Za-z0-9_]{0,63}$/.test(field.name) ||
|
|
seen.has(field.name) ||
|
|
typeof field.type !== 'string' ||
|
|
!FIELD_TYPES.includes(field.type as OutputFieldType)
|
|
) {
|
|
return undefined;
|
|
}
|
|
seen.add(field.name);
|
|
fields.push({ name: field.name, type: field.type as OutputFieldType, required: true });
|
|
}
|
|
return fields;
|
|
}
|
|
|
|
interface StructureBody {
|
|
project_id?: unknown;
|
|
document_id?: unknown;
|
|
version_id?: unknown;
|
|
checksum_sha256?: unknown;
|
|
pipeline_stage?: unknown;
|
|
instruction?: unknown;
|
|
fields?: unknown;
|
|
model_id?: unknown;
|
|
data_class?: unknown;
|
|
format?: unknown;
|
|
}
|
|
|
|
/**
|
|
* POST /api/0pt/documents/structure
|
|
*
|
|
* Upload → DOC0 segments → INF0 model → mechanically validated, evidence-bound
|
|
* output (optionally exported). Credits are reserved before the model runs and
|
|
* settled to the measured usage afterwards.
|
|
*
|
|
* Nothing is trusted from the client except the requested target fields and
|
|
* instruction. Source text is re-read from DOC0 and every returned value must
|
|
* point at an exact span in that text.
|
|
*/
|
|
export async function POST(request: NextRequest) {
|
|
const verified = resolveVerifiedScope(request);
|
|
if ('response' in verified) return verified.response;
|
|
const { tenantId, actorId } = verified.scope;
|
|
|
|
const runtimeConfig = readDocumentWorkbenchRuntime();
|
|
const gateway = createInf0Gateway();
|
|
if (!runtimeConfig.doc0 || !gateway) {
|
|
return NextResponse.json(
|
|
{
|
|
error: 'WORKBENCH_NOT_CONFIGURED',
|
|
message: 'Document structuring is not available until DOC0 and INF0 are configured',
|
|
},
|
|
{ status: 503 },
|
|
);
|
|
}
|
|
const policy = modelPolicy();
|
|
if (!policy) {
|
|
return NextResponse.json(
|
|
{
|
|
error: 'MODEL_POLICY_NOT_CONFIGURED',
|
|
message: 'Document structuring is not available until the model policy is configured',
|
|
},
|
|
{ status: 503 },
|
|
);
|
|
}
|
|
// Explicit pilot switch: `OPT_CREDITS_MODE=off` runs structuring without a
|
|
// wallet. It is deliberately NOT a silent fallback — the response reports
|
|
// `credits: null` and the operator has named the mode.
|
|
const creditsEnabled =
|
|
process.env.OPT_CREDITS_MODE !== 'off' &&
|
|
Boolean(readCreditPriceBook()) &&
|
|
Boolean(repositoryDatabaseUrl());
|
|
const priceBook = creditsEnabled ? readCreditPriceBook() : undefined;
|
|
const databaseUrl = creditsEnabled ? repositoryDatabaseUrl() : undefined;
|
|
if (!creditsEnabled && process.env.OPT_CREDITS_MODE !== 'off') {
|
|
return NextResponse.json(
|
|
{
|
|
error: 'CREDITS_NOT_CONFIGURED',
|
|
message: 'Document structuring is not available until credits are configured',
|
|
},
|
|
{ status: 503 },
|
|
);
|
|
}
|
|
|
|
let body: StructureBody;
|
|
try {
|
|
body = (await request.json()) as StructureBody;
|
|
} catch {
|
|
return NextResponse.json({ error: 'INVALID_BODY', message: 'JSON body required' }, { status: 400 });
|
|
}
|
|
|
|
const projectId = body.project_id;
|
|
const documentId = body.document_id;
|
|
const versionId = body.version_id;
|
|
const checksum = body.checksum_sha256;
|
|
const pipelineStage = body.pipeline_stage;
|
|
const instruction = body.instruction;
|
|
const modelId = body.model_id;
|
|
const dataClass = body.data_class ?? 'low';
|
|
const format = body.format ?? 'json';
|
|
|
|
if (pipelineStage !== 'parsed') {
|
|
return NextResponse.json(
|
|
{
|
|
error: 'DOCUMENT_NOT_PARSED',
|
|
message:
|
|
'The document has no verified text layer yet (parse_failed or not parsed). OCR is required before structuring.',
|
|
},
|
|
{ status: 422 },
|
|
);
|
|
}
|
|
if (
|
|
!isValidScopeId(projectId) ||
|
|
!isValidScopeId(documentId) ||
|
|
!isValidScopeId(versionId) ||
|
|
typeof checksum !== 'string' ||
|
|
!/^[0-9a-f]{64}$/.test(checksum) ||
|
|
typeof instruction !== 'string' ||
|
|
!instruction.trim() ||
|
|
instruction.length > 4000 ||
|
|
typeof modelId !== 'string' ||
|
|
!policy.allowedModelIds.includes(modelId) ||
|
|
typeof dataClass !== 'string' ||
|
|
!(DATA_CLASSES as readonly string[]).includes(dataClass) ||
|
|
(format !== 'json' && format !== 'csv')
|
|
) {
|
|
return NextResponse.json(
|
|
{ error: 'INVALID_REQUEST', message: 'Invalid document structure request' },
|
|
{ status: 400 },
|
|
);
|
|
}
|
|
const fields = parseFields(body.fields);
|
|
if (!fields) {
|
|
return NextResponse.json(
|
|
{ error: 'INVALID_FIELDS', message: 'A valid field contract is required' },
|
|
{ status: 400 },
|
|
);
|
|
}
|
|
|
|
const authorize = createProjectAuthorizer();
|
|
if (!(await authorize({ tenantId, projectId, actorId }))) {
|
|
return NextResponse.json(
|
|
{ error: 'OPT_ACCESS_DENIED', message: 'Project access is not authorized' },
|
|
{ status: 403 },
|
|
);
|
|
}
|
|
|
|
const correlationId = request.headers.get('x-request-id') ?? crypto.randomUUID();
|
|
const context = {
|
|
tenantId,
|
|
projectId,
|
|
actorId,
|
|
capability: '0pt.document.structure.v1',
|
|
correlationId,
|
|
idempotencyKey: correlationId,
|
|
};
|
|
|
|
let sources;
|
|
try {
|
|
sources = await new Doc0SourceClient({
|
|
baseUrl: runtimeConfig.doc0.baseUrl,
|
|
serviceToken: runtimeConfig.doc0.serviceToken,
|
|
}).read({
|
|
tenantId,
|
|
projectId,
|
|
actorId: runtimeConfig.doc0.actorId ?? actorId,
|
|
requestId: correlationId,
|
|
receipt: {
|
|
documentId,
|
|
versionId,
|
|
checksumSha256: checksum,
|
|
pipelineStage: 'parsed',
|
|
},
|
|
});
|
|
} catch (error: unknown) {
|
|
if (error instanceof Doc0SourceUnavailableError) {
|
|
return NextResponse.json(
|
|
{
|
|
error: 'DOC0_SOURCE_UNAVAILABLE',
|
|
message:
|
|
'The extracted source could not be verified (version, checksum, page cap or tenant). Review or reconciliation is required.',
|
|
},
|
|
{ status: 422 },
|
|
);
|
|
}
|
|
return NextResponse.json(
|
|
{ error: 'DOC0_SOURCE_UNAVAILABLE', message: 'The extracted source is unavailable' },
|
|
{ status: 502 },
|
|
);
|
|
}
|
|
|
|
console.log('[0pt-workbench] doc0 source read:', sources.sources.length, 'segments');
|
|
const sourceChars = sources.sources.reduce((total, source) => total + source.text.length, 0);
|
|
const estimatedInputTokens = Math.ceil(JSON.stringify(sources.sources).length / 4) + 1024;
|
|
if (estimatedInputTokens > policy.maxInputTokens) {
|
|
return NextResponse.json(
|
|
{
|
|
error: 'SOURCE_TOO_LARGE',
|
|
message: 'The document is larger than the configured model input limit',
|
|
},
|
|
{ status: 413 },
|
|
);
|
|
}
|
|
|
|
const maximumCredits =
|
|
creditsEnabled && priceBook
|
|
? creditsForUsage(priceBook, {
|
|
pages: Math.max(1, Math.ceil(sourceChars / 1800)),
|
|
inputTokens: estimatedInputTokens,
|
|
outputTokens: policy.maxOutputTokens,
|
|
})
|
|
: 0;
|
|
const reservationId = crypto.randomUUID();
|
|
const ledger =
|
|
creditsEnabled && databaseUrl ? new PostgresCreditLedger(getRepositorySql(databaseUrl)) : undefined;
|
|
if (ledger) {
|
|
try {
|
|
await ensureTrialCredits(ledger, tenantId, readTrialCredits());
|
|
await ledger.apply({
|
|
tenantId,
|
|
idempotencyKey: `reserve:${reservationId}`,
|
|
kind: 'reserve',
|
|
reference: reservationId,
|
|
amount: maximumCredits,
|
|
});
|
|
} catch (error: unknown) {
|
|
const message = error instanceof Error ? error.message : 'Credit reservation failed';
|
|
if (/Insufficient available credits/i.test(message)) {
|
|
return NextResponse.json(
|
|
{
|
|
error: 'INSUFFICIENT_CREDITS',
|
|
message: 'Not enough credits for this job',
|
|
required_credits: maximumCredits,
|
|
},
|
|
{ status: 402 },
|
|
);
|
|
}
|
|
return NextResponse.json({ error: 'CREDITS_UNAVAILABLE', message }, { status: 503 });
|
|
}
|
|
}
|
|
|
|
try {
|
|
const result = await structureDocument({
|
|
gateway,
|
|
request: {
|
|
context,
|
|
admission: {
|
|
tenantId,
|
|
projectId,
|
|
jobId: `doc:${documentId}:${correlationId}`,
|
|
modelId,
|
|
dataClass,
|
|
maxInputTokens: policy.maxInputTokens,
|
|
maxOutputTokens: policy.maxOutputTokens,
|
|
maximumCredits,
|
|
},
|
|
policy: {
|
|
tenantId,
|
|
projectId,
|
|
allowedModelIds: policy.allowedModelIds,
|
|
allowedDataClasses: policy.allowedDataClasses,
|
|
destination: policy.destination,
|
|
externalTransferAllowed: policy.externalTransferAllowed,
|
|
maxInputTokens: policy.maxInputTokens,
|
|
maxOutputTokens: policy.maxOutputTokens,
|
|
},
|
|
reservation: ledger
|
|
? {
|
|
tenantId,
|
|
projectId,
|
|
jobId: `doc:${documentId}:${correlationId}`,
|
|
status: 'reserved',
|
|
credits: maximumCredits,
|
|
}
|
|
: undefined,
|
|
},
|
|
countTokens: async (messages) => Math.ceil(JSON.stringify(messages).length / 4),
|
|
instructions: instruction,
|
|
fields,
|
|
sources: sources.sources,
|
|
});
|
|
|
|
const charged =
|
|
ledger && priceBook
|
|
? Math.min(
|
|
maximumCredits,
|
|
creditsForUsage(priceBook, {
|
|
pages: Math.max(1, Math.ceil(sourceChars / 1800)),
|
|
inputTokens: result.model.promptTokens,
|
|
outputTokens: result.model.completionTokens,
|
|
}),
|
|
)
|
|
: 0;
|
|
let balance;
|
|
if (ledger) {
|
|
try {
|
|
balance = (
|
|
await ledger.apply({
|
|
tenantId,
|
|
idempotencyKey: `settle:${reservationId}`,
|
|
kind: 'settle',
|
|
reference: reservationId,
|
|
amount: charged,
|
|
})
|
|
).balance;
|
|
} catch {
|
|
// The model result is real; keep the reservation for reconciliation
|
|
// rather than reporting a balance we could not write.
|
|
balance = undefined;
|
|
}
|
|
}
|
|
|
|
if (result.status !== 'mechanically_validated') {
|
|
return NextResponse.json(
|
|
{
|
|
status: 'needs_review',
|
|
issues: result.issues,
|
|
model: result.model,
|
|
credits: ledger ? { reserved: maximumCredits, charged, balance } : null,
|
|
},
|
|
{ status: 200 },
|
|
);
|
|
}
|
|
|
|
const exported =
|
|
format === 'json'
|
|
? exportValidatedOutput({
|
|
format: 'json',
|
|
fields,
|
|
sources: sources.sources,
|
|
value: result.value,
|
|
evidence: result.evidence,
|
|
})
|
|
: exportValidatedOutput({
|
|
format: 'csv',
|
|
fields,
|
|
sources: sources.sources,
|
|
value: result.value,
|
|
evidence: result.evidence,
|
|
});
|
|
|
|
return NextResponse.json(
|
|
{
|
|
status: 'mechanically_validated',
|
|
value: result.value,
|
|
evidence: result.evidence,
|
|
model: result.model,
|
|
credits: ledger ? { reserved: maximumCredits, charged, balance } : null,
|
|
export:
|
|
exported.status === 'ready'
|
|
? { media_type: exported.mediaType, content: exported.content }
|
|
: undefined,
|
|
},
|
|
{ status: 200 },
|
|
);
|
|
} catch (error: unknown) {
|
|
if (error instanceof Inf0OutcomeUnknownError) {
|
|
// Never release a reservation whose model call may have completed.
|
|
return NextResponse.json(
|
|
{
|
|
status: 'outcome_unknown',
|
|
message:
|
|
'The model call may have completed. The credit reservation is kept until its usage is reconciled.',
|
|
reservation_id: reservationId,
|
|
},
|
|
{ status: 202 },
|
|
);
|
|
}
|
|
if (ledger) {
|
|
try {
|
|
await ledger.apply({
|
|
tenantId,
|
|
idempotencyKey: `release:${reservationId}`,
|
|
kind: 'release',
|
|
reference: reservationId,
|
|
amount: 0,
|
|
});
|
|
} catch {
|
|
/* the reservation stays for reconciliation */
|
|
}
|
|
}
|
|
const message = error instanceof Error ? error.message : 'Structuring failed';
|
|
console.error('[0pt-workbench] structure failed:', message);
|
|
return NextResponse.json(
|
|
{ error: 'STRUCTURE_FAILED', message, reservation_id: reservationId },
|
|
{ status: 502 },
|
|
);
|
|
}
|
|
}
|