From 8b571cf2f478ac8f86feb416005b148f4dee8327 Mon Sep 17 00:00:00 2001
From: sax3l <simon@siax.io>
Date: Mon, 2 Nov 2026 17:51:09 +0100
Subject: [PATCH] fix(decision-ollama): use Ollama's native JSON mode without
 thinking

The 9B model spent its whole 4000-token budget on reasoning and came back
model_incomplete. Call /api/chat with format=json, think=false and num_predict,
and read prompt_eval_count/eval_count/done_reason. Structured extraction no
longer needs chain-of-thought.
---
 .../decision-ollama-client.ts                 | 38 ++++++++++++-------
 .../decision-ollama-client.test.ts            | 28 +++++++++-----
 2 files changed, 44 insertions(+), 22 deletions(-)

diff --git a/src/document-workbench/decision-ollama-client.ts b/src/document-workbench/decision-ollama-client.ts
index 5b27de7..bdbb4a6 100644
--- a/src/document-workbench/decision-ollama-client.ts
+++ b/src/document-workbench/decision-ollama-client.ts
@@ -100,7 +100,7 @@ export class DecisionOllamaGatewayClient {
 
     let response: Response;
     try {
-      response = await this.fetchImpl(`${this.baseUrl}/v1/chat/completions`, {
+      response = await this.fetchImpl(`${this.baseUrl}/api/chat`, {
         method: 'POST',
         redirect: 'error',
         signal: AbortSignal.timeout(this.options.timeoutMs ?? 120_000),
@@ -113,9 +113,20 @@ export class DecisionOllamaGatewayClient {
         body: JSON.stringify({
           model: this.options.model,
           messages,
-          max_tokens: admission.maxOutputTokens,
-          temperature: 0,
           stream: false,
+          // Structured extraction: JSON mode, no chain-of-thought output, and a
+          // hard output budget. A thinking model otherwise spends the whole
+          // budget on reasoning and returns model_incomplete.
+          format: 'json',
+          think: false,
+          options: {
+            temperature: 0,
+            num_predict: admission.maxOutputTokens,
+            num_ctx: Math.max(
+              4096,
+              Math.min(131072, admission.maxInputTokens + admission.maxOutputTokens + 1024),
+            ),
+          },
         }),
       });
     } catch {
@@ -131,13 +142,15 @@ export class DecisionOllamaGatewayClient {
     }
     const result = decoded as {
       model?: unknown;
-      choices?: Array<{ message?: { content?: unknown }; finish_reason?: unknown }>;
-      usage?: { prompt_tokens?: unknown; completion_tokens?: unknown; total_tokens?: unknown };
+      message?: { content?: unknown };
+      done?: unknown;
+      done_reason?: unknown;
+      prompt_eval_count?: unknown;
+      eval_count?: unknown;
     } | null;
-    const content = result?.choices?.[0]?.message?.content;
-    const promptTokens = result?.usage?.prompt_tokens;
-    const completionTokens = result?.usage?.completion_tokens;
-    const totalTokens = result?.usage?.total_tokens;
+    const content = result?.message?.content;
+    const promptTokens = result?.prompt_eval_count;
+    const completionTokens = result?.eval_count;
     if (
       !result ||
       typeof content !== 'string' ||
@@ -145,16 +158,15 @@ export class DecisionOllamaGatewayClient {
       !result.model.trim() ||
       !Number.isSafeInteger(promptTokens) ||
       !Number.isSafeInteger(completionTokens) ||
-      !Number.isSafeInteger(totalTokens) ||
       (promptTokens as number) < 0 ||
       (promptTokens as number) > admission.maxInputTokens ||
-      (completionTokens as number) > admission.maxOutputTokens ||
-      (promptTokens as number) + (completionTokens as number) !== totalTokens
+      (completionTokens as number) > admission.maxOutputTokens
     ) {
       throw new Inf0OutcomeUnknownError();
     }
+    const totalTokens = (promptTokens as number) + (completionTokens as number);
 
-    const finish = result.choices?.[0]?.finish_reason;
+    const finish = result.done_reason;
     return {
       status:
         finish === 'stop' && (completionTokens as number) > 0 && content.trim()
diff --git a/tests/document-workbench/decision-ollama-client.test.ts b/tests/document-workbench/decision-ollama-client.test.ts
index 2d090e2..a4516ed 100644
--- a/tests/document-workbench/decision-ollama-client.test.ts
+++ b/tests/document-workbench/decision-ollama-client.test.ts
@@ -55,14 +55,21 @@ describe('decision server Ollama gateway', () => {
     expect(() => assertTrustedDecisionOllamaUrl('http://user:pass@100.96.133.104:11434')).toThrow();
   });
 
-  it('maps an OpenAI-compatible Ollama response into the INF0 receipt', async () => {
-    const c = client(async () =>
-      Response.json({
+  it('maps a native Ollama chat response into the INF0 receipt', async () => {
+    const c = client(async (url, init) => {
+      expect(String(url)).toMatch(/\/api\/chat$/);
+      const body = JSON.parse(String(init?.body));
+      expect(body.format).toBe('json');
+      expect(body.think).toBe(false);
+      return Response.json({
         model: 'hermes-qwen3.5:9b',
-        choices: [{ message: { content: '{"value":{}}' }, finish_reason: 'stop' }],
-        usage: { prompt_tokens: 12, completion_tokens: 8, total_tokens: 20 },
-      }),
-    );
+        message: { role: 'assistant', content: '{"value":{}}' },
+        done: true,
+        done_reason: 'stop',
+        prompt_eval_count: 12,
+        eval_count: 8,
+      });
+    });
     await expect(
       c.complete({ context, admission, policy, reservation: undefined, inputTokens: 12, messages }),
     ).resolves.toMatchObject({
@@ -77,8 +84,11 @@ describe('decision server Ollama gateway', () => {
     const c = client(async () =>
       Response.json({
         model: 'hermes-qwen3.5:9b',
-        choices: [{ message: { content: 'x' }, finish_reason: 'stop' }],
-        usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 99 },
+        message: { role: 'assistant', content: 'x' },
+        done: true,
+        done_reason: 'stop',
+        prompt_eval_count: 1,
+        eval_count: 5000,
       }),
     );
     await expect(
-- 
2.54.0