Fix/original sie export (#279)
* feat: cloud backup to Google Drive + full-archive all-scope Adds a cloud-backup extension that uploads a full-company backup ZIP to the user's own Google Drive via OAuth (drive.file scope only). Refresh tokens are AES-256-GCM encrypted before being stored in extension_data. The full-archive export gains a scope=all mode for whole-company backups (per-period SIE under sie/, per-period rapporter/ subfolders, flat dokument/ manifest tagged with fiscal_period_id). An 80 MB size guard short-circuits generation before the platform response limit. Also fixes a latent bug in lib/core/audit/audit-service.ts where the parameter was named userId while the query filtered by company_id; the audit-trail API route was passing user.id so audit queries returned empty unless user and company shared a UUID. Drive-by: scope the dashboard "fresh start" localStorage key per companyId so dismissing the setup checklist in one company no longer carries over to others. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix: address review comments on cloud backup + archive export - Extend audit trail to_date to end-of-day so last-day entries aren't silently excluded from period-scoped archives. - Apply 413 size-limit guard regardless of include_documents, using the overhead-only figure when documents are excluded. - Use crypto.randomUUID() for Drive multipart boundary to eliminate any collision risk with ZIP payload bytes. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix: migrate legacy setup-gate localStorage keys on dashboard Users who previously dismissed the setup checklist via the old global erp_setup_fresh_start or erp_checklist_dismissed keys were re-gated after the switch to a company-scoped key. Fall back to the legacy keys on read and migrate them to the scoped key on first hit. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix: update customer email handling and anonymization rules in supportmail-to-ticket skill * test: update audit trail to_date expectation for end-of-day timestamp Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix: enhance full archive generation to include SIE source files and master data dumps * Update lib/reports/full-archive-export.ts Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
This commit is contained in:
Mattsson
co-authored by
Claude Opus 4.7
greptile-apps[bot]
Claude Opus 4.7
greptile-apps[bot]
parent
23664e79cb
commit
c0b74d50e4
@@ -307,6 +307,117 @@ describe('generateFullArchive', () => {
|
||||
generateFullArchive(supabase as any, 'company-1', { scope: 'all' })
|
||||
).rejects.toThrow('No fiscal periods found')
|
||||
})
|
||||
|
||||
it('includes imported SIE source files and master-data dumps in all-mode', async () => {
|
||||
const importRow = {
|
||||
id: 'import-1',
|
||||
filename: 'original.se',
|
||||
file_hash: 'abc123',
|
||||
file_storage_path: 'company-1/import-1.se',
|
||||
org_number: '5560000000',
|
||||
company_name: 'Test AB',
|
||||
sie_type: 4,
|
||||
fiscal_year_start: '2024-01-01',
|
||||
fiscal_year_end: '2024-12-31',
|
||||
accounts_count: 42,
|
||||
transactions_count: 120,
|
||||
status: 'completed',
|
||||
fiscal_period_id: PERIOD_2024.id,
|
||||
imported_at: '2024-11-01T10:00:00Z',
|
||||
created_at: '2024-11-01T09:55:00Z',
|
||||
}
|
||||
|
||||
enqueueMany([
|
||||
{ data: COMPANY_ROW }, // fetchCompany
|
||||
{ data: [PERIOD_2024] }, // fetchAllPeriods
|
||||
{ data: [] }, // document_attachments
|
||||
{ data: [importRow] }, // sie_imports
|
||||
{ data: [{ source_account: '9999', target_account: '1510' }] }, // sie_account_mappings
|
||||
{ data: [{ id: 'cust-1', name: 'Acme AB' }] }, // customers
|
||||
{ data: [{ id: 'sup-1', name: 'Supplier AB' }] }, // suppliers
|
||||
{ data: [{ id: 'inv-1', invoice_number: 'F-001' }] }, // invoices
|
||||
{ data: [] }, // invoice_items
|
||||
{ data: [] }, // invoice_payments
|
||||
{ data: [] }, // supplier_invoices
|
||||
{ data: [] }, // supplier_invoice_items
|
||||
{ data: [] }, // receipts
|
||||
{ data: [] }, // receipt_line_items
|
||||
{ data: [] }, // transactions
|
||||
{ data: [] }, // mapping_rules
|
||||
{ data: [] }, // categorization_templates
|
||||
{ data: [] }, // bank_file_imports
|
||||
{ data: [COMPANY_ROW] }, // company_settings
|
||||
])
|
||||
|
||||
const buffer = await generateFullArchive(supabase as any, 'company-1', {
|
||||
scope: 'all',
|
||||
})
|
||||
const zip = await JSZip.loadAsync(buffer)
|
||||
|
||||
const originalFile = zip.file('sie/original/import-1_original.se')
|
||||
expect(originalFile).not.toBeNull()
|
||||
|
||||
const manifestFile = zip.file('sie/original/manifest.json')
|
||||
expect(manifestFile).not.toBeNull()
|
||||
const manifest = JSON.parse(await manifestFile!.async('text'))
|
||||
expect(manifest[0].import_id).toBe('import-1')
|
||||
expect(manifest[0].status).toBe('downloaded')
|
||||
|
||||
const imports = JSON.parse(await zip.file('sie/imports.json')!.async('text'))
|
||||
expect(imports[0].filename).toBe('original.se')
|
||||
expect(zip.file('sie/account_mappings.json')).not.toBeNull()
|
||||
|
||||
expect(zip.file('data/customers.json')).not.toBeNull()
|
||||
expect(zip.file('data/suppliers.json')).not.toBeNull()
|
||||
expect(zip.file('data/invoices.json')).not.toBeNull()
|
||||
expect(zip.file('data/invoice_items.json')).not.toBeNull()
|
||||
expect(zip.file('data/invoice_payments.json')).not.toBeNull()
|
||||
expect(zip.file('data/supplier_invoices.json')).not.toBeNull()
|
||||
expect(zip.file('data/supplier_invoice_items.json')).not.toBeNull()
|
||||
expect(zip.file('data/receipts.json')).not.toBeNull()
|
||||
expect(zip.file('data/receipt_line_items.json')).not.toBeNull()
|
||||
expect(zip.file('data/transactions.json')).not.toBeNull()
|
||||
expect(zip.file('data/mapping_rules.json')).not.toBeNull()
|
||||
expect(zip.file('data/categorization_templates.json')).not.toBeNull()
|
||||
expect(zip.file('data/bank_file_imports.json')).not.toBeNull()
|
||||
expect(zip.file('data/company_settings.json')).not.toBeNull()
|
||||
|
||||
const customers = JSON.parse(await zip.file('data/customers.json')!.async('text'))
|
||||
expect(customers).toEqual([{ id: 'cust-1', name: 'Acme AB' }])
|
||||
})
|
||||
|
||||
it('skips raw SIE blobs when include_documents is false but keeps metadata', async () => {
|
||||
enqueueMany([
|
||||
{ data: COMPANY_ROW },
|
||||
{ data: [PERIOD_2024] },
|
||||
{
|
||||
data: [
|
||||
{
|
||||
id: 'import-1',
|
||||
filename: 'x.se',
|
||||
file_hash: 'h',
|
||||
file_storage_path: 'company-1/import-1.se',
|
||||
status: 'completed',
|
||||
imported_at: '2024-11-01T10:00:00Z',
|
||||
created_at: '2024-11-01T09:55:00Z',
|
||||
},
|
||||
],
|
||||
}, // sie_imports
|
||||
{ data: [] }, // sie_account_mappings
|
||||
])
|
||||
|
||||
const buffer = await generateFullArchive(supabase as any, 'company-1', {
|
||||
scope: 'all',
|
||||
include_documents: false,
|
||||
})
|
||||
const zip = await JSZip.loadAsync(buffer)
|
||||
|
||||
expect(zip.file('sie/imports.json')).not.toBeNull()
|
||||
expect(zip.file('sie/account_mappings.json')).not.toBeNull()
|
||||
expect(zip.file('sie/original/import-1_x.se')).toBeNull()
|
||||
expect(zip.file('sie/original/manifest.json')).toBeNull()
|
||||
expect(zip.file('data/customers.json')).not.toBeNull()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -336,8 +447,8 @@ describe('estimateArchiveSize', () => {
|
||||
|
||||
expect(result.document_bytes).toBe(3_500_000)
|
||||
expect(result.document_count).toBe(2)
|
||||
// overhead is +5 MB
|
||||
expect(result.total_bytes).toBe(3_500_000 + 5 * 1024 * 1024)
|
||||
// overhead is +8 MB
|
||||
expect(result.total_bytes).toBe(3_500_000 + 8 * 1024 * 1024)
|
||||
})
|
||||
|
||||
it('returns overhead only when no documents in scope', async () => {
|
||||
@@ -350,6 +461,6 @@ describe('estimateArchiveSize', () => {
|
||||
|
||||
expect(result.document_bytes).toBe(0)
|
||||
expect(result.document_count).toBe(0)
|
||||
expect(result.total_bytes).toBe(5 * 1024 * 1024)
|
||||
expect(result.total_bytes).toBe(8 * 1024 * 1024)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -70,7 +70,10 @@ interface PeriodReports {
|
||||
}
|
||||
|
||||
const REPORT_CONCURRENCY = 3
|
||||
const ARCHIVE_OVERHEAD_BYTES = 5 * 1024 * 1024
|
||||
// 5 MB for SIE + reports + audit + system doc, +3 MB headroom for master-data
|
||||
// JSON dumps and raw imported SIE files (the bucket caps each file at 50 MB,
|
||||
// but typical SIE4 files are tens of KB so a few MB covers most companies).
|
||||
const ARCHIVE_OVERHEAD_BYTES = 8 * 1024 * 1024
|
||||
|
||||
/**
|
||||
* Generate a full archive ZIP for a company.
|
||||
@@ -144,6 +147,11 @@ export async function generateFullArchive(
|
||||
await writeDocuments(zip, supabase, companyId, periods, options.scope)
|
||||
}
|
||||
|
||||
if (options.scope === 'all') {
|
||||
await writeSieSourceFiles(zip, supabase, companyId, options.include_documents !== false)
|
||||
await writeMasterData(zip, supabase, companyId)
|
||||
}
|
||||
|
||||
const revision = zip.folder('revision')!
|
||||
|
||||
const auditFilters =
|
||||
@@ -388,6 +396,223 @@ async function writeDocuments(
|
||||
dokument.file('manifest.json', JSON.stringify(manifest, null, 2))
|
||||
}
|
||||
|
||||
interface SieImportRow {
|
||||
id: string
|
||||
filename: string | null
|
||||
file_hash: string | null
|
||||
file_storage_path: string | null
|
||||
org_number: string | null
|
||||
company_name: string | null
|
||||
sie_type: number | null
|
||||
fiscal_year_start: string | null
|
||||
fiscal_year_end: string | null
|
||||
accounts_count: number | null
|
||||
transactions_count: number | null
|
||||
status: string | null
|
||||
fiscal_period_id: string | null
|
||||
imported_at: string | null
|
||||
created_at: string | null
|
||||
}
|
||||
|
||||
interface SieSourceManifestEntry {
|
||||
import_id: string
|
||||
filename: string | null
|
||||
storage_path: string | null
|
||||
sha256_hash: string | null
|
||||
sie_type: number | null
|
||||
fiscal_year_start: string | null
|
||||
fiscal_year_end: string | null
|
||||
imported_at: string | null
|
||||
status: 'downloaded' | 'missing' | 'skipped'
|
||||
zip_file_name: string | null
|
||||
error?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy raw imported SIE files from the `sie-files` storage bucket into the
|
||||
* archive under `sie/original/`. Preserves the byte-identical source that the
|
||||
* user uploaded (vs the `sie/<period>.se` files which gnubok re-generates from
|
||||
* the current journal entries).
|
||||
*
|
||||
* `sie/imports.json` and `sie/account_mappings.json` are written regardless of
|
||||
* `includeFiles` — they're small and critical for reconstructing the import
|
||||
* history. Blob download is gated behind `includeFiles` since the files can be
|
||||
* large and share the documents opt-out.
|
||||
*/
|
||||
async function writeSieSourceFiles(
|
||||
zip: JSZip,
|
||||
supabase: SupabaseClient,
|
||||
companyId: string,
|
||||
includeFiles: boolean
|
||||
): Promise<void> {
|
||||
const sieFolder = zip.folder('sie')!
|
||||
|
||||
try {
|
||||
const imports = await fetchAllRows<SieImportRow>(({ from, to }) =>
|
||||
supabase
|
||||
.from('sie_imports')
|
||||
.select(
|
||||
'id, filename, file_hash, file_storage_path, org_number, company_name, sie_type, fiscal_year_start, fiscal_year_end, accounts_count, transactions_count, status, fiscal_period_id, imported_at, created_at'
|
||||
)
|
||||
.eq('company_id', companyId)
|
||||
.order('created_at', { ascending: true })
|
||||
.range(from, to)
|
||||
)
|
||||
|
||||
sieFolder.file('imports.json', JSON.stringify(imports, null, 2))
|
||||
|
||||
const manifest: SieSourceManifestEntry[] = []
|
||||
|
||||
if (includeFiles && imports.length > 0) {
|
||||
const originalFolder = sieFolder.folder('original')!
|
||||
|
||||
for (const imp of imports) {
|
||||
if (!imp.file_storage_path) {
|
||||
manifest.push({
|
||||
import_id: imp.id,
|
||||
filename: imp.filename,
|
||||
storage_path: null,
|
||||
sha256_hash: imp.file_hash,
|
||||
sie_type: imp.sie_type,
|
||||
fiscal_year_start: imp.fiscal_year_start,
|
||||
fiscal_year_end: imp.fiscal_year_end,
|
||||
imported_at: imp.imported_at,
|
||||
status: 'skipped',
|
||||
zip_file_name: null,
|
||||
error: 'No storage path on record',
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
const zipFileName = `${imp.id}_${sanitizeFileName(imp.filename || `${imp.id}.se`)}`
|
||||
|
||||
try {
|
||||
const { data: fileData, error } = await supabase.storage
|
||||
.from('sie-files')
|
||||
.download(imp.file_storage_path)
|
||||
|
||||
if (error || !fileData) {
|
||||
manifest.push({
|
||||
import_id: imp.id,
|
||||
filename: imp.filename,
|
||||
storage_path: imp.file_storage_path,
|
||||
sha256_hash: imp.file_hash,
|
||||
sie_type: imp.sie_type,
|
||||
fiscal_year_start: imp.fiscal_year_start,
|
||||
fiscal_year_end: imp.fiscal_year_end,
|
||||
imported_at: imp.imported_at,
|
||||
status: 'missing',
|
||||
zip_file_name: null,
|
||||
error: error?.message || 'Download returned no data',
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
const buffer = await fileData.arrayBuffer()
|
||||
originalFolder.file(zipFileName, buffer)
|
||||
manifest.push({
|
||||
import_id: imp.id,
|
||||
filename: imp.filename,
|
||||
storage_path: imp.file_storage_path,
|
||||
sha256_hash: imp.file_hash,
|
||||
sie_type: imp.sie_type,
|
||||
fiscal_year_start: imp.fiscal_year_start,
|
||||
fiscal_year_end: imp.fiscal_year_end,
|
||||
imported_at: imp.imported_at,
|
||||
status: 'downloaded',
|
||||
zip_file_name: zipFileName,
|
||||
})
|
||||
} catch (err) {
|
||||
manifest.push({
|
||||
import_id: imp.id,
|
||||
filename: imp.filename,
|
||||
storage_path: imp.file_storage_path,
|
||||
sha256_hash: imp.file_hash,
|
||||
sie_type: imp.sie_type,
|
||||
fiscal_year_start: imp.fiscal_year_start,
|
||||
fiscal_year_end: imp.fiscal_year_end,
|
||||
imported_at: imp.imported_at,
|
||||
status: 'missing',
|
||||
zip_file_name: null,
|
||||
error: err instanceof Error ? err.message : 'Unknown error',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
originalFolder.file('manifest.json', JSON.stringify(manifest, null, 2))
|
||||
}
|
||||
|
||||
const mappings = await fetchAllRows<Record<string, unknown>>(({ from, to }) =>
|
||||
supabase
|
||||
.from('sie_account_mappings')
|
||||
.select('*')
|
||||
.eq('company_id', companyId)
|
||||
.range(from, to)
|
||||
)
|
||||
sieFolder.file('account_mappings.json', JSON.stringify(mappings, null, 2))
|
||||
} catch {
|
||||
// SIE metadata fetch failed — archive will still contain the re-generated SIE files
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Dump structured master data as JSON under `data/`. These records are implicit
|
||||
* in the SIE export (as journal entries) but not recoverable as domain objects
|
||||
* without this dump — critical for disaster recovery of a company's state.
|
||||
*/
|
||||
async function writeMasterData(
|
||||
zip: JSZip,
|
||||
supabase: SupabaseClient,
|
||||
companyId: string
|
||||
): Promise<void> {
|
||||
const data = zip.folder('data')!
|
||||
|
||||
const tables: Array<{ name: string; file: string; orderBy?: string }> = [
|
||||
{ name: 'customers', file: 'customers.json', orderBy: 'created_at' },
|
||||
{ name: 'suppliers', file: 'suppliers.json', orderBy: 'created_at' },
|
||||
{ name: 'invoices', file: 'invoices.json', orderBy: 'invoice_date' },
|
||||
{ name: 'invoice_items', file: 'invoice_items.json' },
|
||||
{ name: 'invoice_payments', file: 'invoice_payments.json', orderBy: 'payment_date' },
|
||||
{ name: 'supplier_invoices', file: 'supplier_invoices.json', orderBy: 'invoice_date' },
|
||||
{ name: 'supplier_invoice_items', file: 'supplier_invoice_items.json' },
|
||||
{ name: 'receipts', file: 'receipts.json', orderBy: 'receipt_date' },
|
||||
{ name: 'receipt_line_items', file: 'receipt_line_items.json' },
|
||||
{ name: 'transactions', file: 'transactions.json', orderBy: 'booking_date' },
|
||||
{ name: 'mapping_rules', file: 'mapping_rules.json' },
|
||||
{ name: 'categorization_templates', file: 'categorization_templates.json' },
|
||||
{ name: 'bank_file_imports', file: 'bank_file_imports.json', orderBy: 'created_at' },
|
||||
{ name: 'company_settings', file: 'company_settings.json' },
|
||||
]
|
||||
|
||||
await Promise.all(
|
||||
tables.map(async (t) => {
|
||||
try {
|
||||
const rows = await fetchAllRows<Record<string, unknown>>(({ from, to }) => {
|
||||
let q = supabase.from(t.name).select('*').eq('company_id', companyId)
|
||||
if (t.orderBy) {
|
||||
q = q.order(t.orderBy, { ascending: true })
|
||||
}
|
||||
return q.range(from, to)
|
||||
})
|
||||
data.file(t.file, JSON.stringify(rows, null, 2))
|
||||
} catch (err) {
|
||||
data.file(
|
||||
t.file,
|
||||
JSON.stringify(
|
||||
{ error: err instanceof Error ? err.message : 'Fetch failed', rows: [] },
|
||||
null,
|
||||
2
|
||||
)
|
||||
)
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
function sanitizeFileName(name: string): string {
|
||||
return name.replace(/[\\/:*?"<>|]/g, '_').slice(0, 120)
|
||||
}
|
||||
|
||||
async function buildEntryToPeriodMap(
|
||||
supabase: SupabaseClient,
|
||||
companyId: string,
|
||||
|
||||
Reference in New Issue
Block a user