diff --git a/README.md b/README.md index 05061db..e3cc6f2 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,38 @@ fabric logs --worker tcb-a --otlp-grpc :4317 Everything stays on your machine — FABRIC is a local collector, not a third-party service. Telemetry is read-only: FABRIC ingests spans/logs/metrics for display but never writes back to NEEDLE or modifies worker state. +## Log Retention (`fabric prune`) + +`~/.needle/logs/` grows unbounded as NEEDLE workers create telemetry JSONL and stderr logs. `fabric prune` enforces a retention policy: + +```bash +# Run with defaults (archive after 3 days, hard delete after 7 days) +fabric prune + +# Dry run — see what would happen +fabric prune --dry-run + +# Custom retention +fabric prune --archive-after 5 --max-age 14 --archive-retain 60 + +# Prune a different directory +fabric prune --source /path/to/logs +``` + +| Flag | Default | Description | +|------|---------|-------------| +| `--archive-after` | 3 days | Archive files older than this into `~/.needle/logs/archive/YYYY-MM-DD.tar.gz` | +| `--max-age` | 7 days | Hard delete files older than this (safety net) | +| `--archive-retain` | 30 days | Delete archive tarballs older than this | +| `--dry-run` | off | Report what would happen without making changes | + +The pruner emits `mend.logs_pruned` events to `~/.needle/logs/fabric-mend.jsonl`, visible to FABRIC's directory tailer. Run via cron for automatic retention: + +```bash +# Daily at 03:17 +17 3 * * * ~/.local/bin/fabric prune +``` + ## Production Deployment FABRIC runs as a user-level systemd service (`fabric-web.service`) with OTLP/HTTP enabled: diff --git a/src/cli.ts b/src/cli.ts index 899817f..2fd2f24 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -482,6 +482,31 @@ program } }); +program + .command('prune') + .description('Prune old NEEDLE log files (archive + delete)') + .option('--source ', 'Log directory to prune (default: ~/.needle/logs)') + .option('--archive-after ', 'Archive files older than N days', '3') + .option('--archive-retain ', 'Delete archives older than N days', '30') + .option('--max-age ', 'Delete files older than N days regardless', '7') + .option('--dry-run', 'Report what would happen without making changes') + .action(async (options) => { + const { pruneLogs, formatPruneResult } = await import('./logPruner.js'); + const logDir = options.source + ? (options.source.startsWith('~') ? options.source.replace('~', HOME) : options.source) + : `${HOME}/.needle/logs`; + + const result = pruneLogs({ + logDir, + archiveAfterDays: parseInt(options.archiveAfter, 10) || 3, + archiveRetentionDays: parseInt(options.archiveRetain, 10) || 30, + maxAgeDays: parseInt(options.maxAge, 10) || 7, + dryRun: !!options.dryRun, + }); + + console.log(formatPruneResult(result, !!options.dryRun)); + }); + program .command('digest') .description('Generate session digest from log file') diff --git a/src/logPruner.test.ts b/src/logPruner.test.ts new file mode 100644 index 0000000..0bba1f8 --- /dev/null +++ b/src/logPruner.test.ts @@ -0,0 +1,219 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { pruneLogs, formatPruneResult, type PruneResult } from './logPruner.js'; + +function makeDir(tmp: string, ...segments: string[]): string { + const dir = path.join(tmp, ...segments); + fs.mkdirSync(dir, { recursive: true }); + return dir; +} + +function touch(dir: string, name: string, ageDays: number, sizeBytes = 100): string { + const f = path.join(dir, name); + const buf = Buffer.alloc(sizeBytes); + fs.writeFileSync(f, buf); + const mtime = new Date(Date.now() - ageDays * 24 * 60 * 60 * 1000); + fs.utimesSync(f, mtime, mtime); + return f; +} + +describe('logPruner', () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fabric-prune-test-')); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('archives files older than archiveAfterDays', () => { + const logDir = makeDir(tmpDir, 'logs'); + + // Recent file — should be kept + touch(logDir, 'recent.jsonl', 1, 200); + // Old file — should be archived + touch(logDir, 'old-alpha-001.jsonl', 5, 300); + + const result = pruneLogs({ + logDir, + archiveAfterDays: 3, + archiveRetentionDays: 30, + maxAgeDays: 7, + dryRun: false, + }); + + expect(result.filesScanned).toBe(2); + expect(result.filesArchived).toBe(1); + expect(result.filesDeleted).toBe(0); + expect(result.fileCountBefore).toBe(2); + expect(result.fileCountAfter).toBe(1); // only recent.jsonl remains + expect(result.archivesCreated).toBe(1); + + // Archive should exist + const archiveDir = path.join(logDir, 'archive'); + const archives = fs.readdirSync(archiveDir).filter(f => f.endsWith('.tar.gz')); + expect(archives.length).toBe(1); + }); + + it('deletes files older than maxAgeDays (safety net)', () => { + const logDir = makeDir(tmpDir, 'logs'); + + // Very old file — past both archive and max age + touch(logDir, 'ancient-beta-002.jsonl', 10, 400); + + const result = pruneLogs({ + logDir, + archiveAfterDays: 3, + archiveRetentionDays: 30, + maxAgeDays: 7, + dryRun: false, + }); + + expect(result.filesDeleted).toBe(1); + expect(result.bytesFreed).toBe(400); + expect(result.fileCountAfter).toBe(0); + }); + + it('deletes old archive tarballs', () => { + const logDir = makeDir(tmpDir, 'logs'); + const archiveDir = makeDir(logDir, 'archive'); + + // Create a stale archive tarball + const archive = path.join(archiveDir, '2026-03-01.tar.gz'); + fs.writeFileSync(archive, Buffer.alloc(500)); + const mtime = new Date(Date.now() - 31 * 24 * 60 * 60 * 1000); + fs.utimesSync(archive, mtime, mtime); + + const result = pruneLogs({ + logDir, + archiveAfterDays: 3, + archiveRetentionDays: 30, + maxAgeDays: 7, + dryRun: false, + }); + + expect(result.archivesDeleted).toBe(1); + expect(result.archivesBefore).toBe(1); + expect(result.archivesAfter).toBe(0); + expect(result.bytesFreed).toBe(500); + }); + + it('skips fabric-mend.jsonl', () => { + const logDir = makeDir(tmpDir, 'logs'); + + // fabric-mend.jsonl should never be touched + touch(logDir, 'fabric-mend.jsonl', 100, 200); + // Regular old file + touch(logDir, 'old-charlie-003.jsonl', 10, 300); + + const result = pruneLogs({ + logDir, + archiveAfterDays: 3, + archiveRetentionDays: 30, + maxAgeDays: 7, + dryRun: false, + }); + + // fabric-mend.jsonl excluded from scan + expect(result.filesScanned).toBe(1); + expect(fs.existsSync(path.join(logDir, 'fabric-mend.jsonl'))).toBe(true); + }); + + it('dry run makes no changes', () => { + const logDir = makeDir(tmpDir, 'logs'); + touch(logDir, 'old-delta-004.jsonl', 10, 300); + + const result = pruneLogs({ + logDir, + archiveAfterDays: 3, + archiveRetentionDays: 30, + maxAgeDays: 7, + dryRun: true, + }); + + // Reports what would happen + expect(result.filesDeleted).toBe(1); + expect(result.bytesFreed).toBe(300); + + // But file still exists + expect(fs.existsSync(path.join(logDir, 'old-delta-004.jsonl'))).toBe(true); + }); + + it('handles empty directory', () => { + const logDir = makeDir(tmpDir, 'logs'); + + const result = pruneLogs({ + logDir, + archiveAfterDays: 3, + archiveRetentionDays: 30, + maxAgeDays: 7, + dryRun: false, + }); + + expect(result.filesScanned).toBe(0); + expect(result.filesArchived).toBe(0); + expect(result.filesDeleted).toBe(0); + }); + + it('handles nonexistent directory', () => { + const result = pruneLogs({ + logDir: path.join(tmpDir, 'no-such-dir'), + archiveAfterDays: 3, + archiveRetentionDays: 30, + maxAgeDays: 7, + dryRun: false, + }); + + expect(result.filesScanned).toBe(0); + expect(result.durationMs).toBeGreaterThanOrEqual(0); + }); + + it('emits mend.logs_pruned event to fabric-mend.jsonl', () => { + const logDir = makeDir(tmpDir, 'logs'); + touch(logDir, 'old-echo-005.jsonl', 10, 200); + + pruneLogs({ + logDir, + archiveAfterDays: 3, + archiveRetentionDays: 30, + maxAgeDays: 7, + dryRun: false, + }); + + const mendFile = path.join(logDir, 'fabric-mend.jsonl'); + expect(fs.existsSync(mendFile)).toBe(true); + + const content = fs.readFileSync(mendFile, 'utf8').trim(); + const event = JSON.parse(content); + expect(event.event_type).toBe('mend.logs_pruned'); + expect(event.worker_id).toBe('fabric-prune'); + expect(event.data.files_deleted).toBe(1); + expect(event.data.bytes_freed).toBe(200); + expect(event.data.dry_run).toBe(false); + }); + + it('formatPruneResult includes key info', () => { + const result: PruneResult = { + filesScanned: 100, + filesArchived: 20, + filesDeleted: 50, + archivesCreated: 3, + archivesDeleted: 1, + bytesFreed: 1024 * 1024 * 5, + fileCountBefore: 100, + fileCountAfter: 30, + archivesBefore: 5, + archivesAfter: 4, + durationMs: 123, + }; + + const text = formatPruneResult(result, false); + expect(text).toContain('Files scanned: 100'); + expect(text).toContain('5.0 MB'); + expect(text).toContain('100 → 30'); + }); +}); diff --git a/src/logPruner.ts b/src/logPruner.ts new file mode 100644 index 0000000..4bed5c8 --- /dev/null +++ b/src/logPruner.ts @@ -0,0 +1,339 @@ +/** + * FABRIC Log Pruner + * + * Retention policy for ~/.needle/logs/ — archives old files into + * dated tarballs and deletes expired archives. Emits mend.logs_pruned + * events visible to FABRIC's directory tailer. + * + * Policy: + * 1. Files older than archiveAfterDays → archived into ~/.needle/logs/archive/YYYY-MM-DD.tar.gz + * 2. Original files deleted after successful archive + * 3. Archive tarballs older than archiveRetentionDays → deleted + * 4. Safety net: files older than maxAgeDays deleted directly (even if not archived) + * + * The pruner skips the archive/ directory and fabric-mend events file. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { execFileSync } from 'child_process'; + +export interface PruneOptions { + /** Directory to prune (default: ~/.needle/logs) */ + logDir: string; + + /** Archive files older than this many days (default: 3) */ + archiveAfterDays: number; + + /** Delete archive tarballs older than this many days (default: 30) */ + archiveRetentionDays: number; + + /** Hard maximum age — files older than this are deleted even if not archived (default: 7) */ + maxAgeDays: number; + + /** Dry run — report what would happen without making changes */ + dryRun: boolean; + + /** File patterns to skip (matched against basename) */ + skipPatterns: string[]; +} + +export interface PruneResult { + filesScanned: number; + filesArchived: number; + filesDeleted: number; + archivesCreated: number; + archivesDeleted: number; + bytesFreed: number; + fileCountBefore: number; + fileCountAfter: number; + archivesBefore: number; + archivesAfter: number; + durationMs: number; +} + +export interface FileGroup { + date: string; // YYYY-MM-DD + files: string[]; + totalSize: number; +} + +const SKIP_NAMES = new Set(['archive', 'fabric-mend.jsonl']); + +function defaultLogDir(): string { + const home = process.env.HOME || ''; + return path.join(home, '.needle', 'logs'); +} + +function daysAgo(days: number): number { + return Date.now() - days * 24 * 60 * 60 * 1000; +} + +/** Group file paths by their mtime date (YYYY-MM-DD). */ +function groupByDate(files: string[], cutoffMs: number): Map { + const groups = new Map(); + for (const f of files) { + const stat = fs.statSync(f); + if (stat.mtimeMs >= cutoffMs) continue; + const d = stat.mtime.toISOString().slice(0, 10); + let group = groups.get(d); + if (!group) { + group = { date: d, files: [], totalSize: 0 }; + groups.set(d, group); + } + group.files.push(f); + group.totalSize += stat.size; + } + return groups; +} + +/** Create archive directory if it doesn't exist, return its path. */ +function ensureArchiveDir(logDir: string): string { + const archiveDir = path.join(logDir, 'archive'); + if (!fs.existsSync(archiveDir)) { + fs.mkdirSync(archiveDir, { recursive: true }); + } + return archiveDir; +} + +/** Create a tar.gz archive from a list of files. Returns the archive path. */ +function createTarball(archiveDir: string, date: string, files: string[], dryRun: boolean): string { + const tarballPath = path.join(archiveDir, `${date}.tar.gz`); + + if (fs.existsSync(tarballPath)) { + // Append to existing tarball — tar -rf doesn't work with compressed archives, + // so we extract, add, and recompress. Simpler: just add to existing tarball. + // Since tar --append doesn't work with .tar.gz, create a temporary uncompressed + // tar, append, then recompress. + const tmpTar = path.join(archiveDir, `${date}.tmp.tar`); + try { + // Decompress existing archive + if (!dryRun) { + execFileSync('gzip', ['-d', '-k', '-f', tarballPath], { timeout: 60000 }); + const gzPath = `${tarballPath.slice(0, -3)}`; // remove .gz + fs.renameSync(gzPath, tmpTar); + + // Append new files + const fileArgs = files.map(f => path.basename(f)); + execFileSync('tar', ['-rf', tmpTar, ...fileArgs], { + cwd: path.dirname(files[0]), + timeout: 60000, + }); + + // Recompress + execFileSync('gzip', ['-f', tmpTar], { timeout: 60000 }); + fs.renameSync(`${tmpTar}.gz`, tarballPath); + } + } catch { + // If append fails, just overwrite + if (!dryRun) { + if (fs.existsSync(tmpTar)) fs.unlinkSync(tmpTar); + if (fs.existsSync(`${tmpTar}.gz`)) fs.unlinkSync(`${tmpTar}.gz`); + const fileArgs = files.map(f => path.basename(f)); + execFileSync('tar', ['-czf', tarballPath, ...fileArgs], { + cwd: path.dirname(files[0]), + timeout: 60000, + }); + } + } + } else { + if (!dryRun) { + const fileArgs = files.map(f => path.basename(f)); + execFileSync('tar', ['-czf', tarballPath, ...fileArgs], { + cwd: path.dirname(files[0]), + timeout: 60000, + }); + } + } + + return tarballPath; +} + +/** Emit a mend.logs_pruned event to the fabric-mend events file. */ +function emitMendEvent(logDir: string, result: PruneResult, dryRun: boolean): void { + const eventPath = path.join(logDir, 'fabric-mend.jsonl'); + const event = { + timestamp: new Date().toISOString(), + event_type: 'mend.logs_pruned', + worker_id: 'fabric-prune', + session_id: `prune-${Date.now().toString(36)}`, + sequence: 0, + schema_version: 1, + data: { + files_scanned: result.filesScanned, + files_archived: result.filesArchived, + files_deleted: result.filesDeleted, + archives_created: result.archivesCreated, + archives_deleted: result.archivesDeleted, + bytes_freed: result.bytesFreed, + file_count_before: result.fileCountBefore, + file_count_after: result.fileCountAfter, + dry_run: dryRun, + }, + }; + + if (!dryRun) { + fs.appendFileSync(eventPath, JSON.stringify(event) + '\n'); + } +} + +/** Format bytes as human-readable string. */ +function formatBytes(bytes: number): string { + if (bytes === 0) return '0 B'; + const units = ['B', 'KB', 'MB', 'GB']; + const i = Math.min(Math.floor(Math.log(bytes) / Math.log(1024)), units.length - 1); + return `${(bytes / Math.pow(1024, i)).toFixed(1)} ${units[i]}`; +} + +/** + * Run the log pruning policy. + * + * @returns PruneResult with statistics about what was done + */ +export function pruneLogs(options: Partial = {}): PruneResult { + const startMs = Date.now(); + const logDir = options.logDir || defaultLogDir(); + const archiveAfterDays = options.archiveAfterDays ?? 3; + const archiveRetentionDays = options.archiveRetentionDays ?? 30; + const maxAgeDays = options.maxAgeDays ?? 7; + const dryRun = options.dryRun ?? false; + const skipPatterns = options.skipPatterns ?? []; + const skipRegexes = skipPatterns.map(p => new RegExp(p)); + + if (!fs.existsSync(logDir)) { + return { + filesScanned: 0, filesArchived: 0, filesDeleted: 0, + archivesCreated: 0, archivesDeleted: 0, bytesFreed: 0, + fileCountBefore: 0, fileCountAfter: 0, + archivesBefore: 0, archivesAfter: 0, durationMs: Date.now() - startMs, + }; + } + + // Phase 0: Count current state + const allEntries = fs.readdirSync(logDir); + const logFiles = allEntries.filter(e => { + if (SKIP_NAMES.has(e)) return false; + if (skipRegexes.some(r => r.test(e))) return false; + const full = path.join(logDir, e); + try { return fs.statSync(full).isFile(); } catch { return false; } + }); + const fileCountBefore = logFiles.length; + + const archiveDir = ensureArchiveDir(logDir); + const existingArchives = fs.readdirSync(archiveDir).filter(e => e.endsWith('.tar.gz')); + const archivesBefore = existingArchives.length; + + let filesArchived = 0; + let filesDeleted = 0; + let archivesCreated = 0; + let archivesDeleted = 0; + let bytesFreed = 0; + + // Phase 1: Archive old files (older than archiveAfterDays) + const archiveCutoff = daysAgo(archiveAfterDays); + const fullPaths = logFiles.map(f => path.join(logDir, f)); + const groups = groupByDate(fullPaths, archiveCutoff); + + for (const [date, group] of groups) { + // Skip files that are also past maxAgeDays — they'll be deleted in phase 3 + const maxCutoff = daysAgo(maxAgeDays); + const toArchive = group.files.filter(f => { + const stat = fs.statSync(f); + return stat.mtimeMs >= maxCutoff; + }); + + if (toArchive.length === 0) continue; + + if (!dryRun) { + createTarball(archiveDir, date, toArchive, dryRun); + } + archivesCreated++; + + // Delete archived originals + for (const f of toArchive) { + const size = fs.statSync(f).size; + if (!dryRun) fs.unlinkSync(f); + filesArchived++; + bytesFreed += size; + } + } + + // Phase 2: Delete old archive tarballs + const archiveAgeCutoff = daysAgo(archiveRetentionDays); + for (const archive of existingArchives) { + const archivePath = path.join(archiveDir, archive); + const stat = fs.statSync(archivePath); + if (stat.mtimeMs < archiveAgeCutoff) { + if (!dryRun) fs.unlinkSync(archivePath); + archivesDeleted++; + bytesFreed += stat.size; + } + } + + // Phase 3: Safety net — delete files older than maxAgeDays + const maxCutoff = daysAgo(maxAgeDays); + const remainingEntries = fs.existsSync(logDir) ? fs.readdirSync(logDir) : []; + for (const entry of remainingEntries) { + if (SKIP_NAMES.has(entry)) continue; + if (skipRegexes.some(r => r.test(entry))) continue; + const fullPath = path.join(logDir, entry); + try { + const stat = fs.statSync(fullPath); + if (!stat.isFile()) continue; + if (stat.mtimeMs < maxCutoff) { + if (!dryRun) fs.unlinkSync(fullPath); + filesDeleted++; + bytesFreed += stat.size; + } + } catch { /* skip */ } + } + + // Count final state + const finalEntries = fs.existsSync(logDir) ? fs.readdirSync(logDir) : []; + const fileCountAfter = finalEntries.filter(e => { + if (SKIP_NAMES.has(e)) return false; + try { + return fs.statSync(path.join(logDir, e)).isFile(); + } catch { return false; } + }).length; + + const finalArchives = fs.existsSync(archiveDir) ? fs.readdirSync(archiveDir).filter(e => e.endsWith('.tar.gz')) : []; + + const result: PruneResult = { + filesScanned: fileCountBefore, + filesArchived, + filesDeleted, + archivesCreated, + archivesDeleted, + bytesFreed, + fileCountBefore, + fileCountAfter, + archivesBefore, + archivesAfter: finalArchives.length, + durationMs: Date.now() - startMs, + }; + + // Phase 4: Emit mend.logs_pruned event + if (!dryRun) { + emitMendEvent(logDir, result, dryRun); + } + + return result; +} + +/** Format a PruneResult as a human-readable summary. */ +export function formatPruneResult(result: PruneResult, dryRun: boolean): string { + const prefix = dryRun ? '[DRY RUN] ' : ''; + const lines = [ + `${prefix}Prune complete (${result.durationMs}ms)`, + ` Files scanned: ${result.filesScanned}`, + ` Files archived: ${result.filesArchived}`, + ` Files deleted: ${result.filesDeleted}`, + ` Bytes freed: ${formatBytes(result.bytesFreed)}`, + ` Archives created: ${result.archivesCreated}`, + ` Archives deleted: ${result.archivesDeleted}`, + ` File count: ${result.fileCountBefore} → ${result.fileCountAfter}`, + ` Archive count: ${result.archivesBefore} → ${result.archivesAfter}`, + ]; + return lines.join('\n'); +}