diff --git a/lib/reports/__tests__/full-archive-export.test.ts b/lib/reports/__tests__/full-archive-export.test.ts index ffb77657..a26c03d7 100644 --- a/lib/reports/__tests__/full-archive-export.test.ts +++ b/lib/reports/__tests__/full-archive-export.test.ts @@ -307,6 +307,117 @@ describe('generateFullArchive', () => { generateFullArchive(supabase as any, 'company-1', { scope: 'all' }) ).rejects.toThrow('No fiscal periods found') }) + + it('includes imported SIE source files and master-data dumps in all-mode', async () => { + const importRow = { + id: 'import-1', + filename: 'original.se', + file_hash: 'abc123', + file_storage_path: 'company-1/import-1.se', + org_number: '5560000000', + company_name: 'Test AB', + sie_type: 4, + fiscal_year_start: '2024-01-01', + fiscal_year_end: '2024-12-31', + accounts_count: 42, + transactions_count: 120, + status: 'completed', + fiscal_period_id: PERIOD_2024.id, + imported_at: '2024-11-01T10:00:00Z', + created_at: '2024-11-01T09:55:00Z', + } + + enqueueMany([ + { data: COMPANY_ROW }, // fetchCompany + { data: [PERIOD_2024] }, // fetchAllPeriods + { data: [] }, // document_attachments + { data: [importRow] }, // sie_imports + { data: [{ source_account: '9999', target_account: '1510' }] }, // sie_account_mappings + { data: [{ id: 'cust-1', name: 'Acme AB' }] }, // customers + { data: [{ id: 'sup-1', name: 'Supplier AB' }] }, // suppliers + { data: [{ id: 'inv-1', invoice_number: 'F-001' }] }, // invoices + { data: [] }, // invoice_items + { data: [] }, // invoice_payments + { data: [] }, // supplier_invoices + { data: [] }, // supplier_invoice_items + { data: [] }, // receipts + { data: [] }, // receipt_line_items + { data: [] }, // transactions + { data: [] }, // mapping_rules + { data: [] }, // categorization_templates + { data: [] }, // bank_file_imports + { data: [COMPANY_ROW] }, // company_settings + ]) + + const buffer = await generateFullArchive(supabase as any, 'company-1', { + scope: 'all', + }) + const zip = await JSZip.loadAsync(buffer) + + const originalFile = zip.file('sie/original/import-1_original.se') + expect(originalFile).not.toBeNull() + + const manifestFile = zip.file('sie/original/manifest.json') + expect(manifestFile).not.toBeNull() + const manifest = JSON.parse(await manifestFile!.async('text')) + expect(manifest[0].import_id).toBe('import-1') + expect(manifest[0].status).toBe('downloaded') + + const imports = JSON.parse(await zip.file('sie/imports.json')!.async('text')) + expect(imports[0].filename).toBe('original.se') + expect(zip.file('sie/account_mappings.json')).not.toBeNull() + + expect(zip.file('data/customers.json')).not.toBeNull() + expect(zip.file('data/suppliers.json')).not.toBeNull() + expect(zip.file('data/invoices.json')).not.toBeNull() + expect(zip.file('data/invoice_items.json')).not.toBeNull() + expect(zip.file('data/invoice_payments.json')).not.toBeNull() + expect(zip.file('data/supplier_invoices.json')).not.toBeNull() + expect(zip.file('data/supplier_invoice_items.json')).not.toBeNull() + expect(zip.file('data/receipts.json')).not.toBeNull() + expect(zip.file('data/receipt_line_items.json')).not.toBeNull() + expect(zip.file('data/transactions.json')).not.toBeNull() + expect(zip.file('data/mapping_rules.json')).not.toBeNull() + expect(zip.file('data/categorization_templates.json')).not.toBeNull() + expect(zip.file('data/bank_file_imports.json')).not.toBeNull() + expect(zip.file('data/company_settings.json')).not.toBeNull() + + const customers = JSON.parse(await zip.file('data/customers.json')!.async('text')) + expect(customers).toEqual([{ id: 'cust-1', name: 'Acme AB' }]) + }) + + it('skips raw SIE blobs when include_documents is false but keeps metadata', async () => { + enqueueMany([ + { data: COMPANY_ROW }, + { data: [PERIOD_2024] }, + { + data: [ + { + id: 'import-1', + filename: 'x.se', + file_hash: 'h', + file_storage_path: 'company-1/import-1.se', + status: 'completed', + imported_at: '2024-11-01T10:00:00Z', + created_at: '2024-11-01T09:55:00Z', + }, + ], + }, // sie_imports + { data: [] }, // sie_account_mappings + ]) + + const buffer = await generateFullArchive(supabase as any, 'company-1', { + scope: 'all', + include_documents: false, + }) + const zip = await JSZip.loadAsync(buffer) + + expect(zip.file('sie/imports.json')).not.toBeNull() + expect(zip.file('sie/account_mappings.json')).not.toBeNull() + expect(zip.file('sie/original/import-1_x.se')).toBeNull() + expect(zip.file('sie/original/manifest.json')).toBeNull() + expect(zip.file('data/customers.json')).not.toBeNull() + }) }) }) @@ -336,8 +447,8 @@ describe('estimateArchiveSize', () => { expect(result.document_bytes).toBe(3_500_000) expect(result.document_count).toBe(2) - // overhead is +5 MB - expect(result.total_bytes).toBe(3_500_000 + 5 * 1024 * 1024) + // overhead is +8 MB + expect(result.total_bytes).toBe(3_500_000 + 8 * 1024 * 1024) }) it('returns overhead only when no documents in scope', async () => { @@ -350,6 +461,6 @@ describe('estimateArchiveSize', () => { expect(result.document_bytes).toBe(0) expect(result.document_count).toBe(0) - expect(result.total_bytes).toBe(5 * 1024 * 1024) + expect(result.total_bytes).toBe(8 * 1024 * 1024) }) }) diff --git a/lib/reports/full-archive-export.ts b/lib/reports/full-archive-export.ts index 1f02d846..52d922ba 100644 --- a/lib/reports/full-archive-export.ts +++ b/lib/reports/full-archive-export.ts @@ -70,7 +70,10 @@ interface PeriodReports { } const REPORT_CONCURRENCY = 3 -const ARCHIVE_OVERHEAD_BYTES = 5 * 1024 * 1024 +// 5 MB for SIE + reports + audit + system doc, +3 MB headroom for master-data +// JSON dumps and raw imported SIE files (the bucket caps each file at 50 MB, +// but typical SIE4 files are tens of KB so a few MB covers most companies). +const ARCHIVE_OVERHEAD_BYTES = 8 * 1024 * 1024 /** * Generate a full archive ZIP for a company. @@ -144,6 +147,11 @@ export async function generateFullArchive( await writeDocuments(zip, supabase, companyId, periods, options.scope) } + if (options.scope === 'all') { + await writeSieSourceFiles(zip, supabase, companyId, options.include_documents !== false) + await writeMasterData(zip, supabase, companyId) + } + const revision = zip.folder('revision')! const auditFilters = @@ -388,6 +396,223 @@ async function writeDocuments( dokument.file('manifest.json', JSON.stringify(manifest, null, 2)) } +interface SieImportRow { + id: string + filename: string | null + file_hash: string | null + file_storage_path: string | null + org_number: string | null + company_name: string | null + sie_type: number | null + fiscal_year_start: string | null + fiscal_year_end: string | null + accounts_count: number | null + transactions_count: number | null + status: string | null + fiscal_period_id: string | null + imported_at: string | null + created_at: string | null +} + +interface SieSourceManifestEntry { + import_id: string + filename: string | null + storage_path: string | null + sha256_hash: string | null + sie_type: number | null + fiscal_year_start: string | null + fiscal_year_end: string | null + imported_at: string | null + status: 'downloaded' | 'missing' | 'skipped' + zip_file_name: string | null + error?: string +} + +/** + * Copy raw imported SIE files from the `sie-files` storage bucket into the + * archive under `sie/original/`. Preserves the byte-identical source that the + * user uploaded (vs the `sie/.se` files which gnubok re-generates from + * the current journal entries). + * + * `sie/imports.json` and `sie/account_mappings.json` are written regardless of + * `includeFiles` — they're small and critical for reconstructing the import + * history. Blob download is gated behind `includeFiles` since the files can be + * large and share the documents opt-out. + */ +async function writeSieSourceFiles( + zip: JSZip, + supabase: SupabaseClient, + companyId: string, + includeFiles: boolean +): Promise { + const sieFolder = zip.folder('sie')! + + try { + const imports = await fetchAllRows(({ from, to }) => + supabase + .from('sie_imports') + .select( + 'id, filename, file_hash, file_storage_path, org_number, company_name, sie_type, fiscal_year_start, fiscal_year_end, accounts_count, transactions_count, status, fiscal_period_id, imported_at, created_at' + ) + .eq('company_id', companyId) + .order('created_at', { ascending: true }) + .range(from, to) + ) + + sieFolder.file('imports.json', JSON.stringify(imports, null, 2)) + + const manifest: SieSourceManifestEntry[] = [] + + if (includeFiles && imports.length > 0) { + const originalFolder = sieFolder.folder('original')! + + for (const imp of imports) { + if (!imp.file_storage_path) { + manifest.push({ + import_id: imp.id, + filename: imp.filename, + storage_path: null, + sha256_hash: imp.file_hash, + sie_type: imp.sie_type, + fiscal_year_start: imp.fiscal_year_start, + fiscal_year_end: imp.fiscal_year_end, + imported_at: imp.imported_at, + status: 'skipped', + zip_file_name: null, + error: 'No storage path on record', + }) + continue + } + + const zipFileName = `${imp.id}_${sanitizeFileName(imp.filename || `${imp.id}.se`)}` + + try { + const { data: fileData, error } = await supabase.storage + .from('sie-files') + .download(imp.file_storage_path) + + if (error || !fileData) { + manifest.push({ + import_id: imp.id, + filename: imp.filename, + storage_path: imp.file_storage_path, + sha256_hash: imp.file_hash, + sie_type: imp.sie_type, + fiscal_year_start: imp.fiscal_year_start, + fiscal_year_end: imp.fiscal_year_end, + imported_at: imp.imported_at, + status: 'missing', + zip_file_name: null, + error: error?.message || 'Download returned no data', + }) + continue + } + + const buffer = await fileData.arrayBuffer() + originalFolder.file(zipFileName, buffer) + manifest.push({ + import_id: imp.id, + filename: imp.filename, + storage_path: imp.file_storage_path, + sha256_hash: imp.file_hash, + sie_type: imp.sie_type, + fiscal_year_start: imp.fiscal_year_start, + fiscal_year_end: imp.fiscal_year_end, + imported_at: imp.imported_at, + status: 'downloaded', + zip_file_name: zipFileName, + }) + } catch (err) { + manifest.push({ + import_id: imp.id, + filename: imp.filename, + storage_path: imp.file_storage_path, + sha256_hash: imp.file_hash, + sie_type: imp.sie_type, + fiscal_year_start: imp.fiscal_year_start, + fiscal_year_end: imp.fiscal_year_end, + imported_at: imp.imported_at, + status: 'missing', + zip_file_name: null, + error: err instanceof Error ? err.message : 'Unknown error', + }) + } + } + + originalFolder.file('manifest.json', JSON.stringify(manifest, null, 2)) + } + + const mappings = await fetchAllRows>(({ from, to }) => + supabase + .from('sie_account_mappings') + .select('*') + .eq('company_id', companyId) + .range(from, to) + ) + sieFolder.file('account_mappings.json', JSON.stringify(mappings, null, 2)) + } catch { + // SIE metadata fetch failed — archive will still contain the re-generated SIE files + } +} + +/** + * Dump structured master data as JSON under `data/`. These records are implicit + * in the SIE export (as journal entries) but not recoverable as domain objects + * without this dump — critical for disaster recovery of a company's state. + */ +async function writeMasterData( + zip: JSZip, + supabase: SupabaseClient, + companyId: string +): Promise { + const data = zip.folder('data')! + + const tables: Array<{ name: string; file: string; orderBy?: string }> = [ + { name: 'customers', file: 'customers.json', orderBy: 'created_at' }, + { name: 'suppliers', file: 'suppliers.json', orderBy: 'created_at' }, + { name: 'invoices', file: 'invoices.json', orderBy: 'invoice_date' }, + { name: 'invoice_items', file: 'invoice_items.json' }, + { name: 'invoice_payments', file: 'invoice_payments.json', orderBy: 'payment_date' }, + { name: 'supplier_invoices', file: 'supplier_invoices.json', orderBy: 'invoice_date' }, + { name: 'supplier_invoice_items', file: 'supplier_invoice_items.json' }, + { name: 'receipts', file: 'receipts.json', orderBy: 'receipt_date' }, + { name: 'receipt_line_items', file: 'receipt_line_items.json' }, + { name: 'transactions', file: 'transactions.json', orderBy: 'booking_date' }, + { name: 'mapping_rules', file: 'mapping_rules.json' }, + { name: 'categorization_templates', file: 'categorization_templates.json' }, + { name: 'bank_file_imports', file: 'bank_file_imports.json', orderBy: 'created_at' }, + { name: 'company_settings', file: 'company_settings.json' }, + ] + + await Promise.all( + tables.map(async (t) => { + try { + const rows = await fetchAllRows>(({ from, to }) => { + let q = supabase.from(t.name).select('*').eq('company_id', companyId) + if (t.orderBy) { + q = q.order(t.orderBy, { ascending: true }) + } + return q.range(from, to) + }) + data.file(t.file, JSON.stringify(rows, null, 2)) + } catch (err) { + data.file( + t.file, + JSON.stringify( + { error: err instanceof Error ? err.message : 'Fetch failed', rows: [] }, + null, + 2 + ) + ) + } + }) + ) +} + +function sanitizeFileName(name: string): string { + return name.replace(/[\\/:*?"<>|]/g, '_').slice(0, 120) +} + async function buildEntryToPeriodMap( supabase: SupabaseClient, companyId: string,