Files
accounted/lib/providers/visma/client.ts
T
Jakob Wennberg a49d75db77 fix(migration): Visma pagination + chunk-insert resilience (the '300 misslyckades' case) (#1455)
* fix(providers): paginate Visma eAccounting with $page/$pagesize

eAccounting silently ignores OData $top/$skip, so every request returned
page 1 and getPaginated appended the first page TotalNumberOfPages times:
customers were imported in triplicate and invoice chunks hit unique
violations. Also stop on an empty page so a stale Meta can never loop or
duplicate.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(migration): survive bad rows in entity imports instead of failing whole chunks

One PostgREST insert per 500-row chunk is all-or-nothing, so a single
duplicate reported every row as failed ('300 misslyckades') with no cause
shown. Now: dedupe repeats within the fetched data (paging faults, source
duplicates), fall back to per-row inserts when a chunk is rejected, store
empty invoice numbers as NULL instead of colliding '', surface the first
DB error in the result UI, and mark all-failed steps with an error icon.
Sales invoices also carry remaining_amount so open invoices no longer
land as settled.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(migration): never per-row retry after a successful bulk insert with short read-back

A succeeded statement whose .select() returns fewer rows than sent means
the rows ARE in the table; retrying them one by one would duplicate every
unreturned row. Pair what came back and report the tail instead.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(migration): count stub-insert casualties as failed and sample enrichment errors

Review follow-ups: invoices dropped because their customer/supplier stub
insert errored are DB failures, not matching misses; classifying them as
noMatch rendered a green result row with the database error hidden.
Enrichment failures now also feed errorSample.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-08-08 10:45:02 +02:00

155 lines
4.4 KiB
TypeScript

import { TokenBucketRateLimiter } from '../rate-limiter';
import { withRetry } from '../retry';
import { VISMA_BASE_URL, VISMA_RATE_LIMIT } from './config';
import { isTimeoutError } from '@/lib/http/fetch-with-timeout';
const FETCH_TIMEOUT_MS = 15_000;
export class VismaApiError extends Error {
constructor(
message: string,
public readonly statusCode: number,
public readonly body?: string,
) {
super(message);
this.name = 'VismaApiError';
}
}
function isRetryableError(error: unknown): boolean {
if (isTimeoutError(error)) return true;
if (error instanceof VismaApiError) {
if (error.statusCode === 401 || error.statusCode === 403 || error.statusCode === 404) {
return false;
}
return error.statusCode === 429 || error.statusCode >= 500;
}
return false;
}
interface VismaPaginatedResponse<T> {
Meta?: { TotalNumberOfPages?: number };
Data: T[];
}
export class VismaClient {
private readonly rateLimiter: TokenBucketRateLimiter;
private readonly baseUrl: string;
constructor(baseUrl?: string) {
this.baseUrl = baseUrl ?? VISMA_BASE_URL;
this.rateLimiter = new TokenBucketRateLimiter(VISMA_RATE_LIMIT, 'ratelimit:visma');
}
async get<T>(accessToken: string, path: string): Promise<T> {
return withRetry(
async () => {
await this.rateLimiter.acquire();
const url = `${this.baseUrl}${path}`;
const response = await fetch(url, {
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
'Content-Type': 'application/json',
},
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
});
if (!response.ok) {
const body = await response.text().catch(() => '');
throw new VismaApiError(
`Visma API error: ${response.status} ${response.statusText}`,
response.status,
body,
);
}
return response.json() as Promise<T>;
},
{
maxAttempts: 3,
initialDelayMs: 1000,
shouldRetry: isRetryableError,
},
);
}
async getPage<T>(
accessToken: string,
path: string,
options?: {
page?: number;
pageSize?: number;
modifiedSince?: string;
modifiedField?: string;
},
): Promise<{ items: T[]; page: number; totalPages: number; totalCount: number }> {
const pageSize = options?.pageSize ?? 100;
const page = options?.page ?? 1;
const params = new URLSearchParams();
// eAccounting paginates with $page/$pagesize (default 50, max 1000).
// OData-style $top/$skip are silently IGNORED by the API: every request
// returns page 1, so a multi-page fetch yields the first page N times
// and never reaches the rest of the collection.
params.set('$page', String(page));
params.set('$pagesize', String(pageSize));
if (options?.modifiedSince && options?.modifiedField) {
params.set(
'$filter',
`${options.modifiedField} gt ${options.modifiedSince}`,
);
}
const separator = path.includes('?') ? '&' : '?';
const fullPath = `${path}${separator}${params.toString()}`;
const response = await this.get<VismaPaginatedResponse<T> & { Meta?: { TotalNumberOfResults?: number; TotalNumberOfPages?: number } }>(accessToken, fullPath);
const totalPages = response.Meta?.TotalNumberOfPages ?? 1;
const totalCount = response.Meta?.TotalNumberOfResults ?? 0;
return {
items: Array.isArray(response.Data) ? response.Data : [],
page,
totalPages,
totalCount,
};
}
async getPaginated<T>(
accessToken: string,
path: string,
options?: {
modifiedSince?: string;
modifiedField?: string;
pageSize?: number;
},
): Promise<T[]> {
const allItems: T[] = [];
let page = 1;
let totalPages = 1;
do {
const result = await this.getPage<T>(accessToken, path, {
page,
pageSize: options?.pageSize,
modifiedSince: options?.modifiedSince,
modifiedField: options?.modifiedField,
});
// An empty page means the collection is exhausted regardless of what
// Meta claims; trusting a stale/buggy TotalNumberOfPages here would
// re-append duplicate rows or loop for nothing.
if (result.items.length === 0) break;
allItems.push(...result.items);
totalPages = result.totalPages;
page++;
} while (page <= totalPages);
return allItems;
}
}