/** * FABRIC Log Pruner * * Retention policy for ~/.needle/logs/ — archives old files into * dated tarballs and deletes expired archives. Emits mend.logs_pruned * events visible to FABRIC's directory tailer. * * Policy: * 1. Files older than archiveAfterDays → archived into ~/.needle/logs/archive/YYYY-MM-DD.tar.gz * 2. Original files deleted after successful archive * 3. Archive tarballs older than archiveRetentionDays → deleted * 4. Safety net: files older than maxAgeDays deleted directly (even if not archived) * * The pruner skips the archive/ directory and fabric-mend events file. */ import * as fs from 'fs'; import * as path from 'path'; import { execFileSync } from 'child_process'; export interface RetentionState { fileCount: number; totalSizeBytes: number; oldestFileAgeDays: number; archiveCount: number; archiveSizeBytes: number; policy: { archiveAfterDays: number; maxAgeDays: number; archiveRetentionDays: number; }; } export interface PruneOptions { /** Directory to prune (default: ~/.needle/logs) */ logDir: string; /** Archive files older than this many days (default: 3) */ archiveAfterDays: number; /** Delete archive tarballs older than this many days (default: 30) */ archiveRetentionDays: number; /** Hard maximum age — files older than this are deleted even if not archived (default: 7) */ maxAgeDays: number; /** Dry run — report what would happen without making changes */ dryRun: boolean; /** File patterns to skip (matched against basename) */ skipPatterns: string[]; } export interface PruneResult { filesScanned: number; filesArchived: number; filesDeleted: number; archivesCreated: number; archivesDeleted: number; bytesFreed: number; fileCountBefore: number; fileCountAfter: number; archivesBefore: number; archivesAfter: number; durationMs: number; retentionState: RetentionState; } export interface FileGroup { date: string; // YYYY-MM-DD files: string[]; totalSize: number; } const SKIP_NAMES = new Set(['archive', 'fabric-mend.jsonl']); function defaultLogDir(): string { const home = process.env.HOME || ''; return path.join(home, '.needle', 'logs'); } function daysAgo(days: number): number { return Date.now() - days * 24 * 60 * 60 * 1000; } /** Group file paths by their mtime date (YYYY-MM-DD). */ function groupByDate(files: string[], cutoffMs: number): Map { const groups = new Map(); for (const f of files) { const stat = fs.statSync(f); if (stat.mtimeMs >= cutoffMs) continue; const d = stat.mtime.toISOString().slice(0, 10); let group = groups.get(d); if (!group) { group = { date: d, files: [], totalSize: 0 }; groups.set(d, group); } group.files.push(f); group.totalSize += stat.size; } return groups; } /** Create archive directory if it doesn't exist, return its path. */ function ensureArchiveDir(logDir: string): string { const archiveDir = path.join(logDir, 'archive'); if (!fs.existsSync(archiveDir)) { fs.mkdirSync(archiveDir, { recursive: true }); } return archiveDir; } /** Create a tar.gz archive from a list of files. Returns the archive path. * If archive exists, extracts it, adds new files, and recreates the tarball. */ function createTarball(archiveDir: string, date: string, files: string[], dryRun: boolean): string { const tarballPath = path.join(archiveDir, `${date}.tar.gz`); const extractDir = path.join(archiveDir, `.tmp-${date}`); const logDir = path.dirname(files[0]); if (!dryRun) { fs.mkdirSync(extractDir, { recursive: true }); // Extract existing archive if present if (fs.existsSync(tarballPath)) { try { execFileSync('tar', ['-xzf', tarballPath, '-C', extractDir], { timeout: 60000 }); } catch { // If extraction fails, archive might be corrupted - start fresh } } // Copy new files into extract dir (preserving relative paths from logDir) for (const f of files) { const destName = path.basename(f); const destPath = path.join(extractDir, destName); // Don't fail if file already exists in archive (same file from same date) if (!fs.existsSync(destPath)) { fs.copyFileSync(f, destPath); } } // Create new tarball from merged content // Use '.' to archive entire directory (avoids ARG_MAX limit with many files) execFileSync('tar', ['-czf', tarballPath, '.'], { cwd: extractDir, timeout: 60000, }); // Clean up extract dir fs.rmSync(extractDir, { recursive: true, force: true }); } return tarballPath; } /** Compute current retention state for a log directory. */ export function computeRetentionState(logDir: string, policy: RetentionState['policy']): RetentionState { const archiveDir = path.join(logDir, 'archive'); let fileCount = 0; let totalSizeBytes = 0; let oldestMtimeMs = Infinity; if (fs.existsSync(logDir)) { for (const entry of fs.readdirSync(logDir)) { if (SKIP_NAMES.has(entry)) continue; const full = path.join(logDir, entry); try { const stat = fs.statSync(full); if (!stat.isFile()) continue; fileCount++; totalSizeBytes += stat.size; if (stat.mtimeMs < oldestMtimeMs) oldestMtimeMs = stat.mtimeMs; } catch { /* skip */ } } } let archiveCount = 0; let archiveSizeBytes = 0; if (fs.existsSync(archiveDir)) { for (const entry of fs.readdirSync(archiveDir)) { if (!entry.endsWith('.tar.gz')) continue; try { const stat = fs.statSync(path.join(archiveDir, entry)); archiveCount++; archiveSizeBytes += stat.size; } catch { /* skip */ } } } const oldestFileAgeDays = oldestMtimeMs === Infinity ? 0 : (Date.now() - oldestMtimeMs) / (24 * 60 * 60 * 1000); return { fileCount, totalSizeBytes, oldestFileAgeDays, archiveCount, archiveSizeBytes, policy }; } /** Emit a mend.logs_pruned event to the fabric-mend events file. */ function emitMendEvent(logDir: string, result: PruneResult, dryRun: boolean): void { const eventPath = path.join(logDir, 'fabric-mend.jsonl'); const event = { timestamp: new Date().toISOString(), event_type: 'mend.logs_pruned', worker_id: 'fabric-prune', session_id: `prune-${Date.now().toString(36)}`, sequence: 0, schema_version: 1, data: { files_scanned: result.filesScanned, files_archived: result.filesArchived, files_deleted: result.filesDeleted, archives_created: result.archivesCreated, archives_deleted: result.archivesDeleted, bytes_freed: result.bytesFreed, file_count_before: result.fileCountBefore, file_count_after: result.fileCountAfter, dry_run: dryRun, retention_state: { file_count: result.retentionState.fileCount, total_size_bytes: result.retentionState.totalSizeBytes, oldest_file_age_days: Math.round(result.retentionState.oldestFileAgeDays * 10) / 10, archive_count: result.retentionState.archiveCount, archive_size_bytes: result.retentionState.archiveSizeBytes, policy: { archive_after_days: result.retentionState.policy.archiveAfterDays, max_age_days: result.retentionState.policy.maxAgeDays, archive_retention_days: result.retentionState.policy.archiveRetentionDays, }, }, }, }; if (!dryRun) { fs.appendFileSync(eventPath, JSON.stringify(event) + '\n'); } } /** Format bytes as human-readable string. */ function formatBytes(bytes: number): string { if (bytes === 0) return '0 B'; const units = ['B', 'KB', 'MB', 'GB']; const i = Math.min(Math.floor(Math.log(bytes) / Math.log(1024)), units.length - 1); return `${(bytes / Math.pow(1024, i)).toFixed(1)} ${units[i]}`; } /** * Run the log pruning policy. * * @returns PruneResult with statistics about what was done */ export function pruneLogs(options: Partial = {}): PruneResult { const startMs = Date.now(); const logDir = options.logDir || defaultLogDir(); const archiveAfterDays = options.archiveAfterDays ?? 3; const archiveRetentionDays = options.archiveRetentionDays ?? 30; const maxAgeDays = options.maxAgeDays ?? 7; const dryRun = options.dryRun ?? false; const skipPatterns = options.skipPatterns ?? []; const skipRegexes = skipPatterns.map(p => new RegExp(p)); const policy = { archiveAfterDays, maxAgeDays, archiveRetentionDays }; if (!fs.existsSync(logDir)) { return { filesScanned: 0, filesArchived: 0, filesDeleted: 0, archivesCreated: 0, archivesDeleted: 0, bytesFreed: 0, fileCountBefore: 0, fileCountAfter: 0, archivesBefore: 0, archivesAfter: 0, durationMs: Date.now() - startMs, retentionState: { fileCount: 0, totalSizeBytes: 0, oldestFileAgeDays: 0, archiveCount: 0, archiveSizeBytes: 0, policy }, }; } // Phase 0: Count current state const allEntries = fs.readdirSync(logDir); const logFiles = allEntries.filter(e => { if (SKIP_NAMES.has(e)) return false; if (skipRegexes.some(r => r.test(e))) return false; const full = path.join(logDir, e); try { return fs.statSync(full).isFile(); } catch { return false; } }); const fileCountBefore = logFiles.length; const archiveDir = ensureArchiveDir(logDir); const existingArchives = fs.readdirSync(archiveDir).filter(e => e.endsWith('.tar.gz')); const archivesBefore = existingArchives.length; let filesArchived = 0; let filesDeleted = 0; let archivesCreated = 0; let archivesDeleted = 0; let bytesFreed = 0; // Phase 1: Archive old files (older than archiveAfterDays) const archiveCutoff = daysAgo(archiveAfterDays); const fullPaths = logFiles.map(f => path.join(logDir, f)); const groups = groupByDate(fullPaths, archiveCutoff); for (const [date, group] of groups) { // Skip files that are also past maxAgeDays — they'll be deleted in phase 3 const maxCutoff = daysAgo(maxAgeDays); const toArchive = group.files.filter(f => { const stat = fs.statSync(f); return stat.mtimeMs >= maxCutoff; }); if (toArchive.length === 0) continue; if (!dryRun) { createTarball(archiveDir, date, toArchive, dryRun); } archivesCreated++; // Delete archived originals for (const f of toArchive) { const size = fs.statSync(f).size; if (!dryRun) fs.unlinkSync(f); filesArchived++; bytesFreed += size; } } // Phase 2: Delete old archive tarballs const archiveAgeCutoff = daysAgo(archiveRetentionDays); for (const archive of existingArchives) { const archivePath = path.join(archiveDir, archive); const stat = fs.statSync(archivePath); if (stat.mtimeMs < archiveAgeCutoff) { if (!dryRun) fs.unlinkSync(archivePath); archivesDeleted++; bytesFreed += stat.size; } } // Phase 3: Safety net — delete files older than maxAgeDays const maxCutoff = daysAgo(maxAgeDays); const remainingEntries = fs.existsSync(logDir) ? fs.readdirSync(logDir) : []; for (const entry of remainingEntries) { if (SKIP_NAMES.has(entry)) continue; if (skipRegexes.some(r => r.test(entry))) continue; const fullPath = path.join(logDir, entry); try { const stat = fs.statSync(fullPath); if (!stat.isFile()) continue; if (stat.mtimeMs < maxCutoff) { if (!dryRun) fs.unlinkSync(fullPath); filesDeleted++; bytesFreed += stat.size; } } catch { /* skip */ } } // Count final state const finalEntries = fs.existsSync(logDir) ? fs.readdirSync(logDir) : []; const fileCountAfter = finalEntries.filter(e => { if (SKIP_NAMES.has(e)) return false; try { return fs.statSync(path.join(logDir, e)).isFile(); } catch { return false; } }).length; const finalArchives = fs.existsSync(archiveDir) ? fs.readdirSync(archiveDir).filter(e => e.endsWith('.tar.gz')) : []; const result: PruneResult = { filesScanned: fileCountBefore, filesArchived, filesDeleted, archivesCreated, archivesDeleted, bytesFreed, fileCountBefore, fileCountAfter, archivesBefore, archivesAfter: finalArchives.length, durationMs: Date.now() - startMs, retentionState: computeRetentionState(logDir, policy), }; // Phase 4: Emit mend.logs_pruned event if (!dryRun) { emitMendEvent(logDir, result, dryRun); } return result; } /** Format a PruneResult as a human-readable summary. */ export function formatPruneResult(result: PruneResult, dryRun: boolean): string { const prefix = dryRun ? '[DRY RUN] ' : ''; const rs = result.retentionState; const lines = [ `${prefix}Prune complete (${result.durationMs}ms)`, ` Files scanned: ${result.filesScanned}`, ` Files archived: ${result.filesArchived}`, ` Files deleted: ${result.filesDeleted}`, ` Bytes freed: ${formatBytes(result.bytesFreed)}`, ` Archives created: ${result.archivesCreated}`, ` Archives deleted: ${result.archivesDeleted}`, ` File count: ${result.fileCountBefore} → ${result.fileCountAfter}`, ` Archive count: ${result.archivesBefore} → ${result.archivesAfter}`, ` Retention state:`, ` Current files: ${rs.fileCount} (${formatBytes(rs.totalSizeBytes)})`, ` Oldest file: ${rs.oldestFileAgeDays.toFixed(1)} days`, ` Archives: ${rs.archiveCount} (${formatBytes(rs.archiveSizeBytes)})`, ` Policy: archive>${rs.policy.archiveAfterDays}d, max>${rs.policy.maxAgeDays}d, retain>${rs.policy.archiveRetentionDays}d`, ]; return lines.join('\n'); }