admin/accounted
3
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Add/db and speed (#1243)

Browse Source 
* fix(privacy): make privacy policy page dark mode friendly

Replace the hardcoded light gradient background with bg-background and
add dark:prose-invert to the prose blocks so body text is readable on
dark cards.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* feat(cloud-backup): sync archives to Dropbox alongside Google Drive

Introduce a CloudStorageProvider interface so performSync builds the
archive set once and talks to storage only through it. Google Drive
keeps its existing behaviour; Dropbox is a second implementation, so
the compliance-relevant half (fingerprints, per-year layout, size
fallback, progressive persistence) cannot drift between targets.

Dropbox uses App folder access, matching the drive.file scope's "only
what the app created" guarantee. Uploads are single-shot under 8 MB and
chunked upload sessions above, every write verified against Dropbox's
content_hash. Call arguments are ASCII-escaped per UTF-16 code unit so
Swedish file names survive the Dropbox-API-Arg header.

Each provider owns its extension_data keys, schedule, failure counter
and alert throttle, so a dead Dropbox token cannot pause a healthy
Drive backup. The google_drive_* keys and the /oauth/callback path are
untouched: both are wire format for already-connected companies.

isConfigured() gates /connect only. A deployment that loses its OAuth
credentials must not trap users with a connection they cannot remove
or a schedule they cannot switch off.

Requires DROPBOX_APP_KEY and DROPBOX_APP_SECRET; the provider row
renders disabled without them. No migration: state is extension_data
JSON throughout.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* fix: remove merge-conflict markers committed in DECISIONS.md

The merge that brought main into this branch staged DECISIONS.md while
it still carried conflict markers, so cdc3a513 shipped an unresolved
hunk (compliance swarm ISO 27001 A.8.32).

DECISIONS.md is an append-only log, so both sides are kept: main's
systemdokumentation entry followed by this branch's Dropbox entries.
No decision was dropped.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Mattsson
2026-07-27 16:49:24 +02:00
committed by GitHub
co-authored by Claude Opus 5
parent 1a7152a7af
commit fbd4b992f5
27 changed files with 2852 additions and 478 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/extensions/cloud-backup/auto-sync/cron/__tests__/route.test.ts
+98 -22
View File
@@ -50,9 +50,9 @@ function makeRequest() {
}
/**
* The route issues two queries against extension_data: schedules
* (key = google_drive_schedule) and connections (key = google_drive_connection,
* with an .in() filter). Route rows to the right result by the `key` eq filter.
* The route issues two queries against extension_data: every provider's
* schedule keys, then the matching connection keys. Both use `.in('key', ...)`,
* so route rows to the right result by inspecting the key list.
*/
function makeSupabaseStub(
scheduleRows: unknown[],
@@ -63,16 +63,18 @@ function makeSupabaseStub(
} = {}
) {
const from = vi.fn().mockImplementation(() => {
let key: string | null = null
let kind: 'schedule' | 'connection' = 'schedule'
const chain: any = {
select: vi.fn().mockReturnThis(),
in: vi.fn().mockReturnThis(),
eq: vi.fn().mockImplementation((column: string, value: string) => {
if (column === 'key') key = value
eq: vi.fn().mockReturnThis(),
in: vi.fn().mockImplementation((column: string, values: string[]) => {
if (column === 'key') {
kind = values.some((v) => v.endsWith('_connection')) ? 'connection' : 'schedule'
}
return chain
}),
then: (resolve: (v: unknown) => void) => {
if (key === 'google_drive_connection') {
if (kind === 'connection') {
return resolve({
data: options.connectionRows ?? [],
error: options.connectionError ?? null,
@@ -93,6 +95,7 @@ function scheduleRow(overrides: Record<string, unknown> = {}) {
return {
company_id: 'c-1',
user_id: 'u-1',
key: 'google_drive_schedule',
value: {
enabled: true,
hour_utc: 12,
@@ -104,6 +107,10 @@ function scheduleRow(overrides: Record<string, unknown> = {}) {
}
}
function connectionRow(companyId: string, value: unknown, key = 'google_drive_connection') {
return { company_id: companyId, key, value }
}
function okSyncResult() {
return {
ok: true as const,
@@ -126,6 +133,12 @@ describe('cloud-backup auto-sync cron', () => {
process.env.NEXT_PUBLIC_SUPABASE_URL = 'https://test.supabase.co'
process.env.SUPABASE_SERVICE_ROLE_KEY = 'service-key'
process.env.NEXT_PUBLIC_APP_URL = 'https://app.test'
// Without credentials the cron deliberately skips a provider, so the
// Google flow has to look configured for these tests to exercise it.
process.env.GOOGLE_CLIENT_ID = 'client-id'
process.env.GOOGLE_CLIENT_SECRET = 'client-secret'
delete process.env.DROPBOX_APP_KEY
delete process.env.DROPBOX_APP_SECRET
mockVerifyCronSecret.mockReturnValue(null)
mockSendBackupFailureAlert.mockResolvedValue({ sent: true })
})
@@ -338,10 +351,7 @@ describe('cloud-backup auto-sync cron', () => {
mockCreateClient.mockReturnValueOnce(
makeSupabaseStub([scheduleRow()], {
connectionRows: [
{
company_id: 'c-1',
value: { status: 'needs_reauth', needs_reauth_at: '2026-07-10T03:00:00.000Z' },
},
connectionRow('c-1', { status: 'needs_reauth', needs_reauth_at: '2026-07-10T03:00:00.000Z' }),
],
})
)
@@ -352,7 +362,7 @@ describe('cloud-backup auto-sync cron', () => {
expect(mockPerformSync).not.toHaveBeenCalled()
expect(body.skipped).toBe(1)
expect(body.results).toEqual([
{ companyId: 'c-1', status: 'skipped', error: 'needs_reauth' },
{ companyId: 'c-1', provider: 'google_drive', status: 'skipped', error: 'needs_reauth' },
])
// The incident had not been alerted yet: one alert, persisted on the schedule.
expect(mockSendBackupFailureAlert).toHaveBeenCalledWith(
@@ -372,10 +382,7 @@ describe('cloud-backup auto-sync cron', () => {
[scheduleRow({ last_alert_at: '2026-07-10T04:00:00.000Z' })],
{
connectionRows: [
{
company_id: 'c-1',
value: { status: 'needs_reauth', needs_reauth_at: '2026-07-10T03:00:00.000Z' },
},
connectionRow('c-1', { status: 'needs_reauth', needs_reauth_at: '2026-07-10T03:00:00.000Z' }),
],
}
)
@@ -398,11 +405,8 @@ describe('cloud-backup auto-sync cron', () => {
],
{
connectionRows: [
{
company_id: 'c-dead',
value: { status: 'needs_reauth', needs_reauth_at: '2026-07-10T03:00:00.000Z' },
},
{ company_id: 'c-live', value: { status: 'active' } },
connectionRow('c-dead', { status: 'needs_reauth', needs_reauth_at: '2026-07-10T03:00:00.000Z' }),
connectionRow('c-live', { status: 'active' }),
],
}
)
@@ -420,6 +424,78 @@ describe('cloud-backup auto-sync cron', () => {
expect(body.successes).toBe(1)
})
it('runs both providers for the same company as independent jobs', async () => {
process.env.DROPBOX_APP_KEY = 'app-key'
process.env.DROPBOX_APP_SECRET = 'app-secret'
mockCreateClient.mockReturnValueOnce(
makeSupabaseStub([
scheduleRow(),
{ ...scheduleRow(), key: 'dropbox_schedule' },
])
)
mockPerformSync.mockResolvedValue(okSyncResult())
const res = await GET(makeRequest())
const body = await res.json()
expect(mockPerformSync).toHaveBeenCalledTimes(2)
const targets = mockPerformSync.mock.calls.map((c) => c[0].provider?.id)
expect(targets).toEqual(['google_drive', 'dropbox'])
// Each writes back to its own schedule record.
const writtenKeys = mockSaveExtensionData.mock.calls.map((c) => c[3])
expect(writtenKeys).toEqual(['google_drive_schedule', 'dropbox_schedule'])
expect(body.successes).toBe(2)
})
it('keeps one provider running when the other holds a dead token', async () => {
process.env.DROPBOX_APP_KEY = 'app-key'
process.env.DROPBOX_APP_SECRET = 'app-secret'
mockCreateClient.mockReturnValueOnce(
makeSupabaseStub(
[scheduleRow(), { ...scheduleRow(), key: 'dropbox_schedule' }],
{
connectionRows: [
connectionRow(
'c-1',
{ status: 'needs_reauth', needs_reauth_at: '2026-07-10T03:00:00.000Z' },
'dropbox_connection'
),
connectionRow('c-1', { status: 'active' }),
],
}
)
)
mockPerformSync.mockResolvedValue(okSyncResult())
const res = await GET(makeRequest())
const body = await res.json()
// A dead Dropbox token must not stop the healthy Drive backup.
expect(mockPerformSync).toHaveBeenCalledTimes(1)
expect(mockPerformSync.mock.calls[0][0].provider?.id).toBe('google_drive')
expect(body.successes).toBe(1)
expect(body.skipped).toBe(1)
expect(mockSendBackupFailureAlert).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ kind: 'needs_reauth', providerLabel: 'Dropbox' })
)
})
it('skips a due schedule whose provider has no credentials here', async () => {
// Dropbox stays unconfigured (see beforeEach): the schedule exists but the
// deployment cannot honour it, so it must not burn the failure counter.
mockCreateClient.mockReturnValueOnce(
makeSupabaseStub([{ ...scheduleRow(), key: 'dropbox_schedule' }])
)
const res = await GET(makeRequest())
const body = await res.json()
expect(mockPerformSync).not.toHaveBeenCalled()
expect(mockSaveExtensionData).not.toHaveBeenCalled()
expect(body.processed).toBe(0)
})
it('fails open and attempts the sync when the connection lookup errors', async () => {
mockCreateClient.mockReturnValueOnce(
makeSupabaseStub([scheduleRow()], {
app/api/extensions/cloud-backup/auto-sync/cron/route.ts
+116 -46
View File
@@ -3,35 +3,37 @@ import { NextResponse } from 'next/server'
import { withCronContext } from '@/lib/api/with-cron-context'
import { errorResponse, errorResponseFromCode } from '@/lib/errors/get-structured-error'
import { getErrorMessage } from '@/lib/errors/get-error-message'
import {
performSync,
CONNECTION_KEY,
SCHEDULE_KEY,
saveExtensionData,
} from '@/extensions/general/cloud-backup/lib/sync'
import { performSync, saveExtensionData } from '@/extensions/general/cloud-backup/lib/sync'
import { isScheduleDue } from '@/extensions/general/cloud-backup/lib/schedule'
import { CLOUD_PROVIDERS } from '@/extensions/general/cloud-backup/lib/provider-registry'
import {
sendBackupFailureAlert,
shouldSendBackupAlert,
type BackupAlertKind,
} from '@/extensions/general/cloud-backup/lib/backup-alert'
import type { CloudStorageProvider } from '@/extensions/general/cloud-backup/lib/cloud-provider'
import type {
GoogleDriveConnection,
GoogleDriveSchedule,
CloudConnection,
CloudSchedule,
} from '@/extensions/general/cloud-backup/types'
/**
* GET /api/extensions/cloud-backup/auto-sync/cron
*
* Runs hourly. Finds all companies whose auto-sync is due (daily slot has
* passed and no attempt has run since it: see `isScheduleDue`) and triggers a
* full Drive backup for each via the shared `performSync()` helper. Companies
* left over when a run hits its time budget stay due and are picked up by the
* next hourly run instead of losing the day.
* Runs hourly. Finds every (company, provider) pair whose auto-sync is due
* (daily slot has passed and no attempt has run since it: see `isScheduleDue`)
* and triggers a full backup for each via the shared `performSync()` helper.
* Pairs left over when a run hits its time budget stay due and are picked up
* by the next hourly run instead of losing the day.
*
* Failures increment `consecutive_failures` on the schedule; alert emails go
* out on dead tokens (once per incident) and repeated failures (threshold in
* `backup-alert.ts`), throttled per company.
* Providers are scheduled independently: a company can back up to Google Drive
* nightly and to Dropbox weekly, and a Dropbox failure never marks the Drive
* backup unhealthy. Each provider keeps its own schedule record, failure
* counter and alert throttle.
*
* Failures increment `consecutive_failures` on that provider's schedule; alert
* emails go out on dead tokens (once per incident) and repeated failures
* (threshold in `backup-alert.ts`), throttled per company and provider.
*
* Uses the service role client: no user session, no RLS. Each row in
* `extension_data` carries its own `user_id` (the user who configured the
@@ -52,11 +54,20 @@ export const GET = withCronContext('cron.cloud_backup_auto_sync', async (_reques
const now = new Date()
const origin = process.env.NEXT_PUBLIC_APP_URL || 'http://localhost:3000'
// Every provider's schedules are fetched, but only providers this deployment
// has credentials for are run: without them every sync would fail on the
// token refresh and burn the failure counter. Schedules belonging to an
// unconfigured provider are counted and logged rather than dropped silently,
// because that is a deployment mistake someone needs to see.
const providerByScheduleKey = new Map<string, CloudStorageProvider>(
CLOUD_PROVIDERS.map((p) => [p.keys.schedule, p])
)
const { data: rows, error } = await supabase
.from('extension_data')
.select('company_id, user_id, value')
.select('company_id, user_id, key, value')
.eq('extension_id', 'cloud-backup')
.eq('key', SCHEDULE_KEY)
.in('key', [...providerByScheduleKey.keys()])
if (error) {
ctx.log.error('failed to fetch schedules', error, {
@@ -70,9 +81,40 @@ export const GET = withCronContext('cron.cloud_backup_auto_sync', async (_reques
return NextResponse.json({ message: 'No schedules configured', processed: 0 })
}
const candidates = rows.filter((r) =>
isScheduleDue(r.value as GoogleDriveSchedule | null, now)
)
interface Candidate {
companyId: string
userId: string
provider: CloudStorageProvider
schedule: CloudSchedule
}
const candidates: Candidate[] = []
const unconfigured = new Map<string, number>()
for (const row of rows) {
const provider = providerByScheduleKey.get(row.key as string)
if (!provider) continue
const schedule = row.value as CloudSchedule | null
if (!isScheduleDue(schedule, now)) continue
if (!provider.isConfigured()) {
unconfigured.set(provider.id, (unconfigured.get(provider.id) ?? 0) + 1)
continue
}
candidates.push({
companyId: row.company_id as string,
userId: row.user_id as string,
provider,
schedule: schedule as CloudSchedule,
})
}
if (unconfigured.size > 0) {
// Companies are expecting a backup that this deployment cannot perform.
ctx.log.error(
'due backups skipped: provider has no OAuth credentials in this environment',
undefined,
{ skipped: Object.fromEntries(unconfigured) }
)
}
if (candidates.length === 0) {
return NextResponse.json({
@@ -83,17 +125,17 @@ export const GET = withCronContext('cron.cloud_backup_auto_sync', async (_reques
}
// Connections flagged needs_reauth carry a permanently dead refresh token
// (Google returned 400 invalid_grant): skip them instead of retrying every
// night. They stay visible in the UI until the user reconnects.
// (the provider returned 400 invalid_grant): skip them instead of retrying
// every night. They stay visible in the UI until the user reconnects.
const connectionKeys = [
...new Set(candidates.map((c) => c.provider.keys.connection)),
]
const { data: connectionRows, error: connectionError } = await supabase
.from('extension_data')
.select('company_id, value')
.select('company_id, key, value')
.eq('extension_id', 'cloud-backup')
.eq('key', CONNECTION_KEY)
.in(
'company_id',
candidates.map((r) => r.company_id as string)
)
.in('key', connectionKeys)
.in('company_id', [...new Set(candidates.map((c) => c.companyId))])
if (connectionError) {
// Fail open: without connection data we cannot tell who needs reauth,
@@ -103,10 +145,12 @@ export const GET = withCronContext('cron.cloud_backup_auto_sync', async (_reques
})
}
const connectionByCompany = new Map<string, GoogleDriveConnection>()
// Keyed by company + connection key: one company can hold a healthy Drive
// connection and a dead Dropbox one at the same time.
const connectionByCompanyAndKey = new Map<string, CloudConnection>()
for (const r of connectionRows ?? []) {
const value = r.value as GoogleDriveConnection | null
if (value) connectionByCompany.set(r.company_id as string, value)
const value = r.value as CloudConnection | null
if (value) connectionByCompanyAndKey.set(`${r.company_id}:${r.key}`, value)
}
const startTime = Date.now()
@@ -114,6 +158,7 @@ export const GET = withCronContext('cron.cloud_backup_auto_sync', async (_reques
const results: {
companyId: string
provider: string
status: 'success' | 'error' | 'skipped'
error?: string
}[] = []
@@ -125,6 +170,7 @@ export const GET = withCronContext('cron.cloud_backup_auto_sync', async (_reques
const maybeAlert = async (params: {
companyId: string
userId: string
providerLabel: string
kind: BackupAlertKind
consecutiveFailures: number
errorMessage: string | null
@@ -144,6 +190,7 @@ export const GET = withCronContext('cron.cloud_backup_auto_sync', async (_reques
const sent = await sendBackupFailureAlert(supabase, {
companyId: params.companyId,
userId: params.userId,
providerLabel: params.providerLabel,
kind: params.kind,
consecutiveFailures: params.consecutiveFailures,
errorMessage: params.errorMessage,
@@ -152,7 +199,7 @@ export const GET = withCronContext('cron.cloud_backup_auto_sync', async (_reques
return sent.sent ? new Date().toISOString() : prior
}
for (const row of candidates) {
for (const candidate of candidates) {
if (Date.now() - startTime > TIME_BUDGET_MS) {
ctx.log.info('time budget reached', {
processedSoFar: results.length,
@@ -161,11 +208,12 @@ export const GET = withCronContext('cron.cloud_backup_auto_sync', async (_reques
break
}
const companyId = row.company_id as string
const userId = row.user_id as string
const schedule = row.value as GoogleDriveSchedule
const { companyId, userId, provider, schedule } = candidate
const scheduleKey = provider.keys.schedule
const connection = connectionByCompany.get(companyId)
const connection = connectionByCompanyAndKey.get(
`${companyId}:${provider.keys.connection}`
)
if (connection?.status === 'needs_reauth') {
// Do not touch last_auto_sync_* here: the schedule keeps showing the
// failure from the night the dead token was detected. But make sure the
@@ -180,21 +228,30 @@ export const GET = withCronContext('cron.cloud_backup_auto_sync', async (_reques
const lastAlertAt = await maybeAlert({
companyId,
userId,
providerLabel: provider.label,
kind: 'needs_reauth',
consecutiveFailures: schedule.consecutive_failures ?? 0,
errorMessage: null,
lastAlertAt: schedule.last_alert_at,
})
if (lastAlertAt !== (schedule.last_alert_at ?? null)) {
await saveExtensionData(supabase, companyId, userId, SCHEDULE_KEY, {
await saveExtensionData(supabase, companyId, userId, scheduleKey, {
...schedule,
last_alert_at: lastAlertAt,
}).catch((persistErr) => {
ctx.log.error('failed to persist alert state', persistErr as Error, { companyId })
ctx.log.error('failed to persist alert state', persistErr as Error, {
companyId,
provider: provider.id,
})
})
}
}
results.push({ companyId, status: 'skipped', error: 'needs_reauth' })
results.push({
companyId,
provider: provider.id,
status: 'skipped',
error: 'needs_reauth',
})
continue
}
@@ -206,6 +263,7 @@ export const GET = withCronContext('cron.cloud_backup_auto_sync', async (_reques
origin,
includeDocuments: true,
allowDocumentFallback: true,
provider,
})
const consecutiveFailures = syncResult.ok
@@ -217,6 +275,7 @@ export const GET = withCronContext('cron.cloud_backup_auto_sync', async (_reques
lastAlertAt = await maybeAlert({
companyId,
userId,
providerLabel: provider.label,
kind: syncResult.reason === 'needs_reauth' ? 'needs_reauth' : 'repeated_failures',
consecutiveFailures,
errorMessage: safeSyncError,
@@ -224,7 +283,7 @@ export const GET = withCronContext('cron.cloud_backup_auto_sync', async (_reques
})
}
const updated: GoogleDriveSchedule = {
const updated: CloudSchedule = {
...schedule,
last_auto_sync_at: new Date().toISOString(),
last_auto_sync_status: syncResult.ok ? 'success' : 'error',
@@ -232,10 +291,11 @@ export const GET = withCronContext('cron.cloud_backup_auto_sync', async (_reques
consecutive_failures: consecutiveFailures,
last_alert_at: lastAlertAt,
}
await saveExtensionData(supabase, companyId, userId, SCHEDULE_KEY, updated)
await saveExtensionData(supabase, companyId, userId, scheduleKey, updated)
results.push({
companyId,
provider: provider.id,
status: syncResult.ok ? 'success' : 'error',
error: safeSyncError ?? undefined,
})
@@ -243,19 +303,21 @@ export const GET = withCronContext('cron.cloud_backup_auto_sync', async (_reques
const safeMessage = getErrorMessage(err)
ctx.log.error('cloud backup sync failed for company', err as Error, {
companyId,
provider: provider.id,
})
const consecutiveFailures = (schedule.consecutive_failures ?? 0) + 1
const lastAlertAt = await maybeAlert({
companyId,
userId,
providerLabel: provider.label,
kind: 'repeated_failures',
consecutiveFailures,
errorMessage: safeMessage.slice(0, 200),
lastAlertAt: schedule.last_alert_at,
})
const updated: GoogleDriveSchedule = {
const updated: CloudSchedule = {
...schedule,
last_auto_sync_at: new Date().toISOString(),
last_auto_sync_status: 'error',
@@ -263,13 +325,21 @@ export const GET = withCronContext('cron.cloud_backup_auto_sync', async (_reques
consecutive_failures: consecutiveFailures,
last_alert_at: lastAlertAt,
}
await saveExtensionData(supabase, companyId, userId, SCHEDULE_KEY, updated).catch(
await saveExtensionData(supabase, companyId, userId, scheduleKey, updated).catch(
(persistErr) => {
ctx.log.error('failed to persist failure state', persistErr as Error, { companyId })
ctx.log.error('failed to persist failure state', persistErr as Error, {
companyId,
provider: provider.id,
})
},
)
results.push({ companyId, status: 'error', error: safeMessage })
results.push({
companyId,
provider: provider.id,
status: 'error',
error: safeMessage,
})
}
}