feat(receipt-hunt): keep looking until the mailboxes stop yielding (#1499)

A press fetched at most three receipts, because fetching one means
downloading it and having a model read the PDF, and a backlog of forty
cannot finish inside one serverless invocation. Working through a real
backlog meant pressing the button a dozen times, which is the shape of a
stopgap rather than a feature.

The loop now lives in the page: it keeps asking for another bounded pass
until one comes back with nothing new, which is the honest signal that the
mailboxes hold nothing more for the purchases still open. Progress shows
as it goes and there is a Stop, read inside the loop so it takes effect on
the current pass rather than after every remaining one.

Deliberately not a queue drained by cron. The finest schedule this app
runs is hourly, so a queue would mean pressing a button and waiting an
hour for anything to happen: worse than what it replaces, and a new table
and worker to maintain. The cost of putting the loop in the browser is
that it runs while the tab is open, which is acceptable for work somebody
just asked for.

A cap of 25 passes is a backstop, not a budget: it exists so a pass that
keeps reporting work it never completes cannot run forever.

Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

components/extensions/general/MailConnectionsPanel.tsx

@@ -1,6 +1,6 @@
 'use client'
 
-import { useCallback, useEffect, useState } from 'react'
+import { useCallback, useEffect, useRef, useState } from 'react'
 import { useTranslations } from 'next-intl'
 import { Loader2 } from 'lucide-react'
 import { Button } from '@/components/ui/button'
@@ -35,6 +35,13 @@ interface MailConnection {
 
 const BASE = '/api/extensions/ext/mail'
 
+/**
+ * Backstop on the loop. Each pass fetches a few receipts, so this is far more
+ * than any real backlog needs; it exists so a pass that keeps reporting work
+ * it never completes cannot run forever.
+ */
+const MAX_PASSES = 25
+
 export function MailConnectionsPanel() {
   const t = useTranslations('mail')
   const [connections, setConnections] = useState<MailConnection[]>([])
@@ -44,6 +51,10 @@ export function MailConnectionsPanel() {
   const [pendingDisconnect, setPendingDisconnect] = useState<MailConnection | null>(null)
   const [hunting, setHunting] = useState(false)
   const [huntResult, setHuntResult] = useState<HuntResult | null>(null)
+  const [progress, setProgress] = useState<{ passes: number; fetched: number; proposed: number } | null>(null)
+  // Read inside the loop, so pressing Stop takes effect on the current pass
+  // rather than after every remaining pass has run.
+  const stopped = useRef(false)
 
   const load = useCallback(async () => {
     try {
@@ -90,25 +101,60 @@ export function MailConnectionsPanel() {
    * than a serverless function may live, so the honest shape is a pass that
    * ends, says what it found and what is left, and can be pressed again.
    */
+  /**
+   * Keep asking until the mailboxes stop yielding.
+   *
+   * Each request is a bounded pass, because fetching a receipt means
+   * downloading it and having a model read the PDF, which is far too slow to
+   * finish a backlog inside one serverless invocation. The loop lives here
+   * rather than in a queue drained by cron: the finest schedule this app runs
+   * is hourly, so a queue would mean pressing a button and waiting an hour.
+   *
+   * It stops when a pass finds nothing new, which is the honest signal that
+   * the mailboxes hold nothing more for the purchases still open. The cap is a
+   * backstop against a pass that keeps reporting work it cannot finish.
+   */
   async function hunt() {
     setHunting(true)
     setHuntResult(null)
+    stopped.current = false
+
+    let passes = 0
+    let fetched = 0
+    let proposed = 0
+
     try {
-      const response = await fetch('/api/receipt-hunt/run', { method: 'POST' })
-      if (!response.ok) {
-        setHuntResult({ searched: 0, fetched: 0, proposed: 0, remaining: 0, failed: true })
-        return
+      while (!stopped.current && passes < MAX_PASSES) {
+        const response = await fetch('/api/receipt-hunt/run', { method: 'POST' })
+        if (!response.ok) {
+          setHuntResult({ searched: 0, fetched, proposed, remaining: 0, failed: true })
+          return
+        }
+
+        const body = (await response.json()) as { data: HuntResult }
+        passes++
+        fetched += body.data.fetched
+        proposed += body.data.proposed
+        setProgress({ passes, fetched, proposed })
+        void load()
+
+        // Nothing new this pass: the mailboxes have no more for what is open.
+        if (body.data.fetched === 0) {
+          setHuntResult({ ...body.data, fetched, proposed })
+          return
+        }
       }
-      const body = (await response.json()) as { data: HuntResult }
-      setHuntResult(body.data)
-      void load()
+
+      setHuntResult({ searched: 0, fetched, proposed, remaining: 0 })
     } catch {
-      setHuntResult({ searched: 0, fetched: 0, proposed: 0, remaining: 0, failed: true })
+      setHuntResult({ searched: 0, fetched, proposed, remaining: 0, failed: true })
     } finally {
       setHunting(false)
+      setProgress(null)
     }
   }
 
+
   async function disconnect(connection: MailConnection) {
     await fetch(`${BASE}/connections?id=${encodeURIComponent(connection.id)}`, { method: 'DELETE' })
     setPendingDisconnect(null)
@@ -171,9 +217,18 @@ export function MailConnectionsPanel() {
                       : t('hunt_none', { left: huntResult.remaining })}
                 </SettingsRowNote>
               ) : null}
+              {hunting ? (
+                <Button variant="ghost" size="sm" onClick={() => { stopped.current = true }}>
+                  {t('hunt_stop')}
+                </Button>
+              ) : null}
               <Button variant="secondary" size="sm" onClick={hunt} disabled={hunting}>
                 {hunting ? <Loader2 className="mr-2 h-4 w-4 animate-spin" /> : null}
-                {hunting ? t('hunt_running') : t('hunt_action')}
+                {hunting
+                  ? progress
+                    ? t('hunt_progress', { fetched: progress.fetched })
+                    : t('hunt_running')
+                  : t('hunt_action')}
               </Button>
             </SettingsRowEnd>
           </SettingsRow>

messages/en.json

@@ -324,6 +324,8 @@
     "sign_in_again": "Sign in again"
   },
   "mail": {
+    "hunt_stop": "Stop",
+    "hunt_progress": "{fetched} fetched…",
     "hunt_title": "Look for receipts",
     "hunt_help": "We search the connected mailboxes for receipts and invoices belonging to purchases without one, read the amount from the file, and put the matches in Review. Nothing is posted.",
     "hunt_row": "Search the mailboxes",

messages/sv.json

@@ -324,6 +324,8 @@
     "sign_in_again": "Logga in igen"
   },
   "mail": {
+    "hunt_stop": "Stoppa",
+    "hunt_progress": "{fetched} hämtade…",
     "hunt_title": "Leta efter underlag",
     "hunt_help": "Vi söker i de kopplade brevlådorna efter kvitton och fakturor till köp som saknar underlag, läser beloppet ur filen och lägger fram förslagen i Granskning. Inget bokförs.",
     "hunt_row": "Sök igenom brevlådorna",