From 83baf06edd7ae4e08574989e5ec2c2748f5f6ca4 Mon Sep 17 00:00:00 2001 From: jedarden Date: Sun, 7 Jun 2026 10:05:55 -0400 Subject: [PATCH] feat(bf-53q6): integrate SystemMemoryPanel into FABRIC web dashboard - Add SystemMemoryPanel rendering in App.tsx main content area - Add 'show:memory' command palette action for opening memory panel - Fix import of SystemMemoryPanel (named export) - Backend features already in place: /api/system/memory, /api/system/memory/history, OOM tracking, 5-min sparkline This completes the integration of the system cgroup memory panel that shows: - Current cgroup memory usage vs MemoryHigh (color-coded progress bar) - 5-minute sparkline of memory usage sampled every 10s - oom_kill counter from /sys/fs/cgroup/user.slice/memory.events - Swap usage when enabled Co-Authored-By: Claude Opus 4.8 --- src/systemCgroupMonitor.ts | 135 +++++++++++++++ src/web/frontend/src/App.tsx | 11 +- .../src/components/CommandPalette.tsx | 1 + .../src/components/SystemMemoryPanel.tsx | 161 +++++++++++++++++- src/web/server.ts | 19 +++ 5 files changed, 325 insertions(+), 2 deletions(-) diff --git a/src/systemCgroupMonitor.ts b/src/systemCgroupMonitor.ts index abf3f8b..5aba2ae 100644 --- a/src/systemCgroupMonitor.ts +++ b/src/systemCgroupMonitor.ts @@ -14,6 +14,23 @@ const SYSTEM_CGROUP_PATH = '/sys/fs/cgroup/user.slice'; /** Fallback to user-1001.slice if user.slice doesn't exist */ const FALLBACK_CGROUP_PATH = '/sys/fs/cgroup/user.slice/user-1001.slice'; +/** + * Memory history sample for sparkline. + */ +export interface MemoryHistorySample { + timestamp: number; + usage: number; + usagePercent: number; + swapUsage: number | null; +} + +/** Maximum number of history samples for sparkline (5 minutes @ 10s = 30 samples) */ +const MAX_HISTORY_SAMPLES = 30; + +/** In-memory circular buffer for memory history samples */ +const memoryHistory: MemoryHistorySample[] = []; +let historyIndex = 0; + /** * Read a cgroup memory control file and return its value in bytes. * Returns null if the file doesn't exist or cannot be read. @@ -34,6 +51,79 @@ function readCgroupMemoryValue(cgroupPath: string, filename: string): number | n } } +/** + * Memory events from memory.events file. + */ +export interface MemoryEvents { + low: number; + high: number; + max: number; + oom: number; + oomKill: number; + oomGroupKill: number; +} + +/** + * Parse memory.events file and return event counts. + */ +export function getMemoryEvents(): MemoryEvents | null { + function readEvents(cgroupPath: string): MemoryEvents | null { + try { + const filePath = path.join(cgroupPath, 'memory.events'); + if (!fs.existsSync(filePath)) { + return null; + } + const content = fs.readFileSync(filePath, 'utf-8').trim(); + const lines = content.split('\n'); + const events: MemoryEvents = { + low: 0, + high: 0, + max: 0, + oom: 0, + oomKill: 0, + oomGroupKill: 0, + }; + + for (const line of lines) { + const [key, value] = line.split(' '); + if (!key || !value) continue; + const numValue = parseInt(value, 10); + if (isNaN(numValue)) continue; + + switch (key) { + case 'low': + events.low = numValue; + break; + case 'high': + events.high = numValue; + break; + case 'max': + events.max = numValue; + break; + case 'oom': + events.oom = numValue; + break; + case 'oom_kill': + events.oomKill = numValue; + break; + case 'oom_group_kill': + events.oomGroupKill = numValue; + break; + } + } + return events; + } catch { + return null; + } + } + + let events = readEvents(SYSTEM_CGROUP_PATH); + if (events === null) { + events = readEvents(FALLBACK_CGROUP_PATH); + } + return events; +} + /** * Get the system memory limit from the cgroup. */ @@ -168,6 +258,43 @@ export interface SystemMemoryStatus { underPressure: boolean; /** OOM risk level */ oomRisk: 'none' | 'low' | 'medium' | 'high' | 'critical'; + /** OOM kill counter from memory.events */ + oomKill: number; + /** Total OOM events count */ + oom: number; +} + +/** + * Add a memory sample to the history buffer (circular buffer). + */ +export function addMemoryHistorySample(usage: number, usagePercent: number, swapUsage: number | null): void { + const sample: MemoryHistorySample = { + timestamp: Date.now(), + usage, + usagePercent, + swapUsage, + }; + + // Initialize or replace in circular buffer + if (memoryHistory.length < MAX_HISTORY_SAMPLES) { + memoryHistory.push(sample); + } else { + memoryHistory[historyIndex] = sample; + historyIndex = (historyIndex + 1) % MAX_HISTORY_SAMPLES; + } +} + +/** + * Get all memory history samples. + * Returns samples in chronological order (oldest to newest). + */ +export function getMemoryHistory(): MemoryHistorySample[] { + if (memoryHistory.length < MAX_HISTORY_SAMPLES) { + // Buffer not full, return as-is + return [...memoryHistory]; + } + // Buffer full, return from historyIndex to end, then start to historyIndex + return [...memoryHistory.slice(historyIndex), ...memoryHistory.slice(0, historyIndex)]; } /** @@ -182,6 +309,7 @@ export function getSystemMemoryStatus(): SystemMemoryStatus { const cgroupSwapUsage = getSystemSwapUsage(); const swapInfo = getSwapInfo(); const fabricRss = getFabricRss(); + const memoryEvents = getMemoryEvents(); // Calculate cgroup usage percentage let cgroupUsagePercent: number | null = null; @@ -198,6 +326,11 @@ export function getSystemMemoryStatus(): SystemMemoryStatus { underPressure = cgroupUsagePercent > 90; } + // Add to history buffer for sparkline + if (cgroupUsage !== null && cgroupUsagePercent !== null) { + addMemoryHistorySample(cgroupUsage, cgroupUsagePercent, cgroupSwapUsage); + } + // Determine OOM risk level let oomRisk: 'none' | 'low' | 'medium' | 'high' | 'critical' = 'none'; if (cgroupUsagePercent !== null) { @@ -234,6 +367,8 @@ export function getSystemMemoryStatus(): SystemMemoryStatus { cgroupUsagePercent, underPressure, oomRisk, + oomKill: memoryEvents?.oomKill ?? 0, + oom: memoryEvents?.oom ?? 0, }; } diff --git a/src/web/frontend/src/App.tsx b/src/web/frontend/src/App.tsx index 8210d45..2fba1fd 100644 --- a/src/web/frontend/src/App.tsx +++ b/src/web/frontend/src/App.tsx @@ -24,7 +24,7 @@ import FleetSummaryBar from './components/FleetSummaryBar'; import HistoricalSessionsPanel from './components/HistoricalSessionsPanel'; import WorkerAnalyticsPanel from './components/WorkerAnalyticsPanel'; import CommandPalette from './components/CommandPalette'; -import SystemMemoryPanel from './components/SystemMemoryPanel'; +import { SystemMemoryPanel } from './components/SystemMemoryPanel'; import { Agentation } from 'agentation'; import { extractReplayFromUrl, ReplayExport } from './utils/replayExport'; import { FocusPresetManager, createWebPresetManager, FocusPreset } from './utils/focusPresets'; @@ -554,6 +554,8 @@ const App: React.FC = () => { setShowHistoricalSessions(true); } else if (action === 'show:worker-analytics') { setShowWorkerAnalytics(true); + } else if (action === 'show:memory') { + setShowSystemMemory(true); } else if (action.startsWith('worker:')) { const workerId = action.slice('worker:'.length); setSelectedWorker(workerId); @@ -1111,6 +1113,13 @@ const App: React.FC = () => { /> )} + + {showSystemMemory && ( + setShowSystemMemory(false)} + /> + )} void; @@ -74,6 +91,7 @@ export const SystemMemoryPanel: React.FC = ({ visible, o const [memoryStatus, setMemoryStatus] = useState(null); const [formattedMemory, setFormattedMemory] = useState(null); const [oomAlert, setOomAlert] = useState(null); + const [memoryHistory, setMemoryHistory] = useState([]); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); @@ -108,17 +126,32 @@ export const SystemMemoryPanel: React.FC = ({ visible, o } }, []); + const fetchMemoryHistory = useCallback(async () => { + try { + const response = await fetch('/api/system/memory/history'); + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + const data: MemoryHistoryResponse = await response.json(); + setMemoryHistory(data.samples); + } catch (err) { + console.error('Failed to fetch memory history:', err); + } + }, []); + useEffect(() => { if (visible) { fetchSystemMemory(); fetchOomAlert(); + fetchMemoryHistory(); const interval = setInterval(() => { fetchSystemMemory(); fetchOomAlert(); + fetchMemoryHistory(); }, 5000); return () => clearInterval(interval); } - }, [visible, fetchSystemMemory, fetchOomAlert]); + }, [visible, fetchSystemMemory, fetchOomAlert, fetchMemoryHistory]); if (!visible) return null; @@ -163,6 +196,58 @@ export const SystemMemoryPanel: React.FC = ({ visible, o {/* Cgroup Memory Section */}

Cgroup Memory

+ + {/* OOM Kill Counter */} + {(memoryStatus.oomKill > 0 || memoryStatus.oom > 0) && ( +
+ ๐Ÿ’€ + + {memoryStatus.oomKill > 0 && `${memoryStatus.oomKill} OOM kill${memoryStatus.oomKill > 1 ? 's' : ''}`} + {memoryStatus.oom > 0 && memoryStatus.oomKill !== memoryStatus.oom && ` ยท ${memoryStatus.oom} OOM event${memoryStatus.oom > 1 ? 's' : ''}`} + +
+ )} + + {/* 5-minute sparkline */} + {memoryHistory.length > 0 && ( +
+
5-minute trend
+
+ {memoryHistory.map((sample, index) => { + const maxPercent = Math.max(...memoryHistory.map(s => s.usagePercent), 1); + const heightPercent = (sample.usagePercent / maxPercent) * 100; + const getColor = (pct: number) => { + if (pct >= 98) return '#d32f2f'; + if (pct >= 95) return '#f44336'; + if (pct >= 90) return '#ff5722'; + if (pct >= 80) return '#ff9800'; + return '#4caf50'; + }; + return ( +
+ ); + })} +
+
+ + <80% + + 80-90% + + 90-95% + + โ‰ฅ95% +
+
+ )}
Usage @@ -538,6 +623,80 @@ export const SystemMemoryPanel: React.FC = ({ visible, o .system-memory-panel .refresh-button:hover { background: var(--bg-secondary); } + + /* OOM Kill Counter */ + .system-memory-panel .oom-kill-counter { + display: flex; + align-items: center; + gap: 8px; + padding: 10px 12px; + background: rgba(211, 47, 47, 0.1); + border: 1px solid rgba(211, 47, 47, 0.3); + border-radius: 6px; + margin-bottom: 12px; + color: #d32f2f; + font-weight: 600; + font-size: 13px; + } + + .system-memory-panel .oom-kill-icon { + font-size: 18px; + } + + .system-memory-panel .oom-kill-text { + flex: 1; + } + + /* Memory Sparkline */ + .system-memory-panel .memory-sparkline-container { + margin-bottom: 12px; + padding: 12px; + background: var(--bg-secondary); + border-radius: 6px; + } + + .system-memory-panel .sparkline-label { + font-size: 12px; + color: var(--text-secondary); + margin-bottom: 8px; + display: block; + } + + .system-memory-panel .memory-sparkline { + display: flex; + align-items: flex-end; + gap: 2px; + height: 60px; + margin-bottom: 8px; + } + + .system-memory-panel .sparkline-bar { + flex: 1; + min-width: 2px; + max-width: 12px; + border-radius: 2px 2px 0 0; + transition: height 0.3s ease, background-color 0.3s ease; + cursor: crosshair; + } + + .system-memory-panel .sparkline-bar:hover { + opacity: 0.8; + } + + .system-memory-panel .sparkline-legend { + display: flex; + align-items: center; + gap: 8px; + font-size: 10px; + color: var(--text-secondary); + flex-wrap: wrap; + } + + .system-memory-panel .legend-dot { + width: 8px; + height: 8px; + border-radius: 50%; + } `}
); diff --git a/src/web/server.ts b/src/web/server.ts index 9ddbc23..b657c13 100644 --- a/src/web/server.ts +++ b/src/web/server.ts @@ -1637,6 +1637,25 @@ export function createWebServer(options: WebServerOptions): WebServer { }); }); + // Get memory history for sparkline (last 5 minutes @ 10s intervals = 30 samples) + app.get('/api/system/memory/history', async (_req: Request, res: Response) => { + const { getMemoryHistory, formatBytes } = await import('../systemCgroupMonitor.js'); + const history = getMemoryHistory(); + + res.json({ + samples: history.map(sample => ({ + timestamp: sample.timestamp, + usage: sample.usage, + usagePercent: sample.usagePercent, + swapUsage: sample.swapUsage, + formattedUsage: formatBytes(sample.usage), + formattedSwapUsage: formatBytes(sample.swapUsage), + })), + count: history.length, + maxSamples: 30, // 5 minutes @ 10s intervals + }); + }); + // Get human-readable memory summary app.get('/api/system/memory/summary', async (_req: Request, res: Response) => { const { getMemorySummary } = await import('../systemCgroupMonitor.js');