fix(documents): record archive integrity checks in their own ledger so the nightly control advances again (#2108)
The 03:00 WORM verification cron stamped last_integrity_check_at on document_attachments. enforce_period_lock_documents() fires on any UPDATE of a row whose journal entry sits in a closed or locked period, without checking whether the entry link actually changed, so a read-only integrity stamp was rejected. The queue orders last_integrity_check_at ASC NULLS FIRST, so the rejected rows re-sorted to the head every night and the batch became permanently 200/200 blocked. Both call sites discarded the update error, so nothing logged and nothing alerted. Prod state: 34 557 current-version documents, 24 083 never checked, last successful stamp 2026-08-31 03:00, nightly successes already decayed to single digits. Migration 017's enforcement triggers are legally required and never-touch, so this does not narrow the trigger. The verification outcome moves to its own document_integrity_checks table and the cron stops writing document_attachments altogether, which takes the trigger off the write path. The legacy column stays in place. Failures are now counted, logged and reported in the route's summary: the silence is why this went unnoticed for weeks. Claude-Session: https://claude.ai/code/session_016ifKg6Ec67A39oxfGPU1yc Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jakob Wennberg
co-authored by
Claude Opus 5
Claude Opus 5
parent
2c46d69d21
commit
77becf3d65
@@ -1,12 +1,17 @@
|
||||
/**
|
||||
* Tests for the nightly document integrity-verify cron.
|
||||
*
|
||||
* Covers the two production defects fixed in this route:
|
||||
* - the run must fit its budget (maxDuration 300 + batch default 200), and
|
||||
* Covers the production defects fixed in this route:
|
||||
* - the run must fit its budget (maxDuration 300 + batch default 200),
|
||||
* - a document whose storage object cannot be downloaded must surface as an
|
||||
* audit incident (INTEGRITY_FAILURE / DOCUMENT_OBJECT_MISSING) AND get its
|
||||
* last_integrity_check_at stamped so it stops head-blocking the
|
||||
* nulls-first queue every night.
|
||||
* audit incident (INTEGRITY_FAILURE / DOCUMENT_OBJECT_MISSING) AND get a
|
||||
* ledger row so it stops head-blocking the queue,
|
||||
* - the queue and the stamp both live in document_integrity_checks: the route
|
||||
* must never write to document_attachments, whose UPDATE trigger
|
||||
* (enforce_period_lock_documents, migration 017) rejects every document
|
||||
* linked to a closed/locked period and wedged the cron at 200/200 rejected,
|
||||
* - and a rejected ledger or audit write must be counted, logged and reported
|
||||
* rather than discarded, which is why the stall went unnoticed for weeks.
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { createHash } from 'node:crypto'
|
||||
@@ -24,39 +29,36 @@ interface MockDoc {
|
||||
storage_path: string
|
||||
sha256_hash: string
|
||||
file_name: string
|
||||
last_checked_at: string | null
|
||||
}
|
||||
|
||||
const state = {
|
||||
documents: [] as MockDoc[],
|
||||
fetchError: null as { message: string } | null,
|
||||
downloadResults: new Map<string, { data: unknown; error: { message: string } | null }>(),
|
||||
updates: [] as Array<{ values: Record<string, unknown>; id: string }>,
|
||||
rpcCalls: [] as Array<{ fn: string; args: Record<string, unknown> }>,
|
||||
ledgerInserts: [] as Array<Record<string, unknown>>,
|
||||
ledgerInsertError: null as { message: string } | null,
|
||||
auditInserts: [] as Array<Record<string, unknown>>,
|
||||
auditInsertError: null as { message: string } | null,
|
||||
limitCalls: [] as number[],
|
||||
}
|
||||
|
||||
function makeMockClient() {
|
||||
return {
|
||||
rpc: (fn: string, args: Record<string, unknown>) => {
|
||||
state.rpcCalls.push({ fn, args })
|
||||
return Promise.resolve({
|
||||
data: state.fetchError ? null : state.documents,
|
||||
error: state.fetchError,
|
||||
})
|
||||
},
|
||||
from: (table: string) => {
|
||||
if (table === 'document_attachments') {
|
||||
if (table === 'document_integrity_checks') {
|
||||
return {
|
||||
select: () => ({
|
||||
eq: () => ({
|
||||
order: () => ({
|
||||
limit: (n: number) => {
|
||||
state.limitCalls.push(n)
|
||||
return Promise.resolve({ data: state.documents, error: state.fetchError })
|
||||
},
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
update: (values: Record<string, unknown>) => ({
|
||||
eq: (_column: string, id: string) => {
|
||||
state.updates.push({ values, id })
|
||||
return Promise.resolve({ error: null })
|
||||
},
|
||||
}),
|
||||
insert: (row: Record<string, unknown>) => {
|
||||
state.ledgerInserts.push(row)
|
||||
return Promise.resolve({ error: state.ledgerInsertError })
|
||||
},
|
||||
}
|
||||
}
|
||||
if (table === 'audit_log') {
|
||||
@@ -67,6 +69,12 @@ function makeMockClient() {
|
||||
},
|
||||
}
|
||||
}
|
||||
if (table === 'document_attachments') {
|
||||
// The whole point of the fix: a write here runs through
|
||||
// enforce_period_lock_documents() and is rejected for every document
|
||||
// linked to a closed/locked period.
|
||||
throw new Error('the verify cron must never write to document_attachments')
|
||||
}
|
||||
throw new Error(`unexpected table: ${table}`)
|
||||
},
|
||||
storage: {
|
||||
@@ -101,6 +109,7 @@ function makeDoc(overrides: Partial<MockDoc> = {}): MockDoc {
|
||||
storage_path: 'user-1/company-1/inbox/file.pdf',
|
||||
sha256_hash: 'deadbeef',
|
||||
file_name: 'file.pdf',
|
||||
last_checked_at: null,
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
@@ -124,10 +133,11 @@ beforeEach(() => {
|
||||
state.documents = []
|
||||
state.fetchError = null
|
||||
state.downloadResults.clear()
|
||||
state.updates = []
|
||||
state.rpcCalls = []
|
||||
state.ledgerInserts = []
|
||||
state.ledgerInsertError = null
|
||||
state.auditInserts = []
|
||||
state.auditInsertError = null
|
||||
state.limitCalls = []
|
||||
})
|
||||
|
||||
describe('GET /api/documents/verify/cron', () => {
|
||||
@@ -139,23 +149,28 @@ describe('GET /api/documents/verify/cron', () => {
|
||||
const response = await GET(cronRequest())
|
||||
|
||||
expect(response.status).toBe(401)
|
||||
expect(state.limitCalls).toHaveLength(0)
|
||||
expect(state.rpcCalls).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('declares a 300s function budget', () => {
|
||||
expect(maxDuration).toBe(300)
|
||||
})
|
||||
|
||||
it('requests a batch of 200 by default and honors the env override', async () => {
|
||||
it('draws the batch from the ledger queue, 200 by default, env override honored', async () => {
|
||||
await GET(cronRequest())
|
||||
expect(state.limitCalls).toEqual([200])
|
||||
expect(state.rpcCalls).toEqual([
|
||||
{ fn: 'next_documents_for_integrity_check', args: { p_limit: 200 } },
|
||||
])
|
||||
|
||||
process.env.DOCUMENT_VERIFY_BATCH_SIZE = '50'
|
||||
await GET(cronRequest())
|
||||
expect(state.limitCalls).toEqual([200, 50])
|
||||
expect(state.rpcCalls[1]).toEqual({
|
||||
fn: 'next_documents_for_integrity_check',
|
||||
args: { p_limit: 50 },
|
||||
})
|
||||
})
|
||||
|
||||
it('stamps last_integrity_check_at on a successful verification', async () => {
|
||||
it('appends a passed ledger row on a successful verification', async () => {
|
||||
const doc = makeDoc()
|
||||
const hash = registerObject(doc.storage_path, '%PDF-1.4 demo content')
|
||||
state.documents = [{ ...doc, sha256_hash: hash }]
|
||||
@@ -168,15 +183,24 @@ describe('GET /api/documents/verify/cron', () => {
|
||||
verified: 1,
|
||||
failures: 0,
|
||||
missingObjects: 0,
|
||||
writeFailures: 0,
|
||||
errors: 0,
|
||||
})
|
||||
expect(state.updates).toHaveLength(1)
|
||||
expect(state.updates[0].id).toBe(doc.id)
|
||||
expect(state.updates[0].values.last_integrity_check_at).toEqual(expect.any(String))
|
||||
expect(state.ledgerInserts).toHaveLength(1)
|
||||
expect(state.ledgerInserts[0]).toMatchObject({
|
||||
document_id: doc.id,
|
||||
company_id: doc.company_id,
|
||||
result: 'passed',
|
||||
expected_sha256: hash,
|
||||
computed_sha256: hash,
|
||||
storage_path: doc.storage_path,
|
||||
detail: null,
|
||||
})
|
||||
expect(state.ledgerInserts[0].checked_at).toEqual(expect.any(String))
|
||||
expect(state.auditInserts).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('writes an INTEGRITY_FAILURE audit row and still stamps on hash mismatch', async () => {
|
||||
it('writes an INTEGRITY_FAILURE audit row and a hash_mismatch ledger row on mismatch', async () => {
|
||||
const doc = makeDoc({ sha256_hash: 'not-the-real-hash' })
|
||||
registerObject(doc.storage_path, 'tampered content')
|
||||
state.documents = [doc]
|
||||
@@ -186,10 +210,16 @@ describe('GET /api/documents/verify/cron', () => {
|
||||
|
||||
expect(json.failures).toBe(1)
|
||||
expect(json.missingObjects).toBe(0)
|
||||
expect(state.updates).toHaveLength(1)
|
||||
expect(json.writeFailures).toBe(0)
|
||||
expect(state.auditInserts).toHaveLength(1)
|
||||
expect(state.auditInserts[0].action).toBe('INTEGRITY_FAILURE')
|
||||
expect(String(state.auditInserts[0].description)).not.toContain('DOCUMENT_OBJECT_MISSING')
|
||||
expect(state.ledgerInserts).toHaveLength(1)
|
||||
expect(state.ledgerInserts[0]).toMatchObject({
|
||||
document_id: doc.id,
|
||||
result: 'hash_mismatch',
|
||||
expected_sha256: 'not-the-real-hash',
|
||||
})
|
||||
})
|
||||
|
||||
it('verifies via the company-scoped fallback key when a concurrent backfill re-homed the object', async () => {
|
||||
@@ -208,17 +238,18 @@ describe('GET /api/documents/verify/cron', () => {
|
||||
const json = await response.json()
|
||||
|
||||
expect(state.auditInserts).toHaveLength(0)
|
||||
expect(state.updates.map((u) => u.id)).toEqual(['doc-repointed'])
|
||||
expect(state.ledgerInserts.map((row) => row.document_id)).toEqual(['doc-repointed'])
|
||||
expect(json).toEqual({
|
||||
processed: 1,
|
||||
verified: 1,
|
||||
failures: 0,
|
||||
missingObjects: 0,
|
||||
writeFailures: 0,
|
||||
errors: 0,
|
||||
})
|
||||
})
|
||||
|
||||
it('surfaces a missing storage object as an audit incident AND stamps the check', async () => {
|
||||
it('surfaces a missing storage object as an audit incident AND an object_missing ledger row', async () => {
|
||||
const missing = makeDoc({ id: 'doc-missing', storage_path: 'user-1/company-1/gone.pdf' })
|
||||
const healthy = makeDoc({ id: 'doc-healthy', storage_path: 'user-1/company-1/ok.pdf' })
|
||||
const healthyHash = registerObject(healthy.storage_path, 'healthy content')
|
||||
@@ -235,20 +266,28 @@ describe('GET /api/documents/verify/cron', () => {
|
||||
expect(String(audit.description)).toContain('DOCUMENT_OBJECT_MISSING')
|
||||
expect(audit.new_state).toMatchObject({ reason: 'DOCUMENT_OBJECT_MISSING' })
|
||||
|
||||
// Both documents are stamped: the failing one must stop head-blocking
|
||||
// the nulls-first queue, and the healthy one was verified.
|
||||
expect(state.updates.map((u) => u.id).sort()).toEqual(['doc-healthy', 'doc-missing'])
|
||||
// Both documents get a ledger row: the failing one must stop head-blocking
|
||||
// the queue, and the healthy one was verified.
|
||||
expect(state.ledgerInserts.map((row) => row.document_id).sort()).toEqual([
|
||||
'doc-healthy',
|
||||
'doc-missing',
|
||||
])
|
||||
expect(state.ledgerInserts.find((row) => row.document_id === 'doc-missing')).toMatchObject({
|
||||
result: 'object_missing',
|
||||
computed_sha256: null,
|
||||
})
|
||||
|
||||
expect(json).toEqual({
|
||||
processed: 2,
|
||||
verified: 1,
|
||||
failures: 0,
|
||||
missingObjects: 1,
|
||||
writeFailures: 0,
|
||||
errors: 1,
|
||||
})
|
||||
})
|
||||
|
||||
it('does not stamp a missing object when the audit insert fails, so it retries next run', async () => {
|
||||
it('does not write a ledger row when the audit insert fails, so it retries next run', async () => {
|
||||
const missing = makeDoc({ id: 'doc-missing', storage_path: 'user-1/company-1/gone.pdf' })
|
||||
state.documents = [missing]
|
||||
state.auditInsertError = { message: 'insert blocked' }
|
||||
@@ -257,12 +296,49 @@ describe('GET /api/documents/verify/cron', () => {
|
||||
const json = await response.json()
|
||||
|
||||
expect(state.auditInserts).toHaveLength(1)
|
||||
expect(state.updates).toHaveLength(0)
|
||||
expect(state.ledgerInserts).toHaveLength(0)
|
||||
expect(json.missingObjects).toBe(0)
|
||||
expect(json.writeFailures).toBe(1)
|
||||
expect(json.errors).toBe(1)
|
||||
})
|
||||
|
||||
it('returns an error envelope when the document fetch fails', async () => {
|
||||
it('reports a rejected ledger write instead of swallowing it', async () => {
|
||||
const doc = makeDoc()
|
||||
const hash = registerObject(doc.storage_path, 'healthy content')
|
||||
state.documents = [{ ...doc, sha256_hash: hash }]
|
||||
state.ledgerInsertError = { message: 'permission denied for table document_integrity_checks' }
|
||||
|
||||
const response = await GET(cronRequest())
|
||||
const json = await response.json()
|
||||
|
||||
expect(state.ledgerInserts).toHaveLength(1)
|
||||
expect(json).toEqual({
|
||||
processed: 1,
|
||||
verified: 0,
|
||||
failures: 0,
|
||||
missingObjects: 0,
|
||||
writeFailures: 1,
|
||||
errors: 1,
|
||||
})
|
||||
})
|
||||
|
||||
it('counts a rejected audit write on a hash mismatch as a write failure', async () => {
|
||||
const doc = makeDoc({ sha256_hash: 'not-the-real-hash' })
|
||||
registerObject(doc.storage_path, 'tampered content')
|
||||
state.documents = [doc]
|
||||
state.auditInsertError = { message: 'insert blocked' }
|
||||
|
||||
const response = await GET(cronRequest())
|
||||
const json = await response.json()
|
||||
|
||||
expect(state.auditInserts).toHaveLength(1)
|
||||
expect(state.ledgerInserts).toHaveLength(0)
|
||||
expect(json.failures).toBe(0)
|
||||
expect(json.writeFailures).toBe(1)
|
||||
expect(json.errors).toBe(1)
|
||||
})
|
||||
|
||||
it('returns an error envelope when the queue fetch fails', async () => {
|
||||
state.fetchError = { message: 'db down' }
|
||||
|
||||
const response = await GET(cronRequest())
|
||||
|
||||
Reference in New Issue
Block a user