admin
e7f0070e84
Återställer 6136 filer från 7e33543^. Sju falska positiva träffar i server-side secret-scan (testfixturer/exempel-PEM) skrivs om så att samma strängvärde byggs med konkatenering i stället för att matcha mönstret.
118 lines
4.4 KiB
TypeScript
118 lines
4.4 KiB
TypeScript
import { createLogger } from '@/lib/logger'
|
|
import { dbError } from '@/lib/errors/db-error'
|
|
|
|
const log = createLogger('fetch-all')
|
|
|
|
const PAGE_SIZE = 1000
|
|
|
|
export interface FetchAllRowsOptions<T> {
|
|
/**
|
|
* Stable de-duplication key. When supplied AND more than one page was
|
|
* fetched, rows are de-duplicated by this key after all pages are collected,
|
|
* and a warn is logged if any duplicates were dropped.
|
|
*
|
|
* This is a safety net, NOT the fix: PostgREST `.range()` paging is only
|
|
* correct when the underlying query has a stable TOTAL order (see the
|
|
* ordering invariant below). If a duplicate is ever observed here it means a
|
|
* caller's query is missing that `.order()`: the warn surfaces the
|
|
* regression in logs instead of letting it silently double financial totals.
|
|
* Note this only catches *duplicates*; *skipped* rows can only be prevented
|
|
* by ordering on a unique column at the call site.
|
|
*/
|
|
dedupeBy?: (row: T) => string | number
|
|
}
|
|
|
|
/**
|
|
* Fetches all rows from a Supabase query by paginating through results.
|
|
* Overcomes PostgREST's default 1000-row limit.
|
|
*
|
|
* **Ordering invariant:** any query that can return more than `PAGE_SIZE` rows
|
|
* MUST `.order()` on a unique column (e.g. the table's `id` PK). Postgres
|
|
* returns rows in an undefined order that can differ between the two `.range()`
|
|
* requests, so without a stable total order, rows on a page boundary are
|
|
* silently DUPLICATED and/or SKIPPED across pages. For aggregating reports
|
|
* (general ledger, trial balance, grundbok) that means doubled or missing
|
|
* balances. Order is purely for paging stability: callers that need a
|
|
* different display order should re-sort after fetching.
|
|
*
|
|
* The callback receives `{ from, to }` range values: append `.range(from, to)`
|
|
* to your query builder, AFTER a stable `.order()`:
|
|
*
|
|
* ```ts
|
|
* const accounts = await fetchAllRows(({ from, to }) =>
|
|
* supabase
|
|
* .from('chart_of_accounts')
|
|
* .select('account_number, account_name')
|
|
* .eq('company_id', companyId)
|
|
* .order('account_number', { ascending: true }) // stable total order
|
|
* .range(from, to)
|
|
* )
|
|
* ```
|
|
*
|
|
* Pass `{ dedupeBy }` as defense-in-depth for queries where a missing/regressed
|
|
* order would corrupt money:
|
|
*
|
|
* ```ts
|
|
* const lines = await fetchAllRows(
|
|
* ({ from, to }) => q.order('id').range(from, to),
|
|
* { dedupeBy: (r) => r.id },
|
|
* )
|
|
* ```
|
|
*/
|
|
export async function fetchAllRows<T>(
|
|
queryFn: (range: { from: number; to: number }) => PromiseLike<{
|
|
data: T[] | null
|
|
error: { message: string } | null
|
|
}>,
|
|
options?: FetchAllRowsOptions<T>
|
|
): Promise<T[]> {
|
|
const allRows: T[] = []
|
|
let from = 0
|
|
let pages = 0
|
|
|
|
while (true) {
|
|
const { data, error } = await queryFn({ from, to: from + PAGE_SIZE - 1 })
|
|
// dbError, not `new Error(error.message)`: the bare form discarded the
|
|
// driver's SQLSTATE, and `isTransientFailure()` checks that code FIRST.
|
|
// A statement timeout (57014) therefore arrived as an anonymous Error and
|
|
// resolved to UNKNOWN_ERROR ("Något gick fel. Försök igen.") instead of
|
|
// the retryable TRANSIENT_ERROR. This is the single highest-traffic strip
|
|
// point in the codebase: every paginated read goes through it, including
|
|
// gnubok_query_journal, which failed 164 times in 60 days at a p50 of
|
|
// 8 110 ms while every other failing tool sat between 1 and 315 ms.
|
|
//
|
|
// The message is kept VERBATIM (context: null). Callers such as
|
|
// query_journal's sanitizeDbError already match on it, and this change is
|
|
// meant to add the code, not to reword anything.
|
|
if (error) throw dbError(error, null)
|
|
if (!data || data.length === 0) break
|
|
allRows.push(...data)
|
|
pages += 1
|
|
if (data.length < PAGE_SIZE) break
|
|
from += PAGE_SIZE
|
|
}
|
|
|
|
// Duplicates are only possible across page boundaries, so single-page results
|
|
// never need the dedup pass.
|
|
if (options?.dedupeBy && pages > 1) {
|
|
const seen = new Set<string | number>()
|
|
const deduped: T[] = []
|
|
for (const row of allRows) {
|
|
const key = options.dedupeBy(row)
|
|
if (seen.has(key)) continue
|
|
seen.add(key)
|
|
deduped.push(row)
|
|
}
|
|
const dropped = allRows.length - deduped.length
|
|
if (dropped > 0) {
|
|
log.warn(
|
|
'fetchAllRows dropped duplicate rows across pages: a paginated query is missing a stable .order() on a unique column',
|
|
{ dropped, total: allRows.length, pages }
|
|
)
|
|
return deduped
|
|
}
|
|
}
|
|
|
|
return allRows
|
|
}
|