feat(bf-53q6): integrate SystemMemoryPanel into FABRIC web dashboard
- Add SystemMemoryPanel rendering in App.tsx main content area - Add 'show:memory' command palette action for opening memory panel - Fix import of SystemMemoryPanel (named export) - Backend features already in place: /api/system/memory, /api/system/memory/history, OOM tracking, 5-min sparkline This completes the integration of the system cgroup memory panel that shows: - Current cgroup memory usage vs MemoryHigh (color-coded progress bar) - 5-minute sparkline of memory usage sampled every 10s - oom_kill counter from /sys/fs/cgroup/user.slice/memory.events - Swap usage when enabled Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jedarden
parent
77b1cd72c3
commit
83baf06edd
5 changed files with 325 additions and 2 deletions
|
|
@ -14,6 +14,23 @@ const SYSTEM_CGROUP_PATH = '/sys/fs/cgroup/user.slice';
|
|||
/** Fallback to user-1001.slice if user.slice doesn't exist */
|
||||
const FALLBACK_CGROUP_PATH = '/sys/fs/cgroup/user.slice/user-1001.slice';
|
||||
|
||||
/**
|
||||
* Memory history sample for sparkline.
|
||||
*/
|
||||
export interface MemoryHistorySample {
|
||||
timestamp: number;
|
||||
usage: number;
|
||||
usagePercent: number;
|
||||
swapUsage: number | null;
|
||||
}
|
||||
|
||||
/** Maximum number of history samples for sparkline (5 minutes @ 10s = 30 samples) */
|
||||
const MAX_HISTORY_SAMPLES = 30;
|
||||
|
||||
/** In-memory circular buffer for memory history samples */
|
||||
const memoryHistory: MemoryHistorySample[] = [];
|
||||
let historyIndex = 0;
|
||||
|
||||
/**
|
||||
* Read a cgroup memory control file and return its value in bytes.
|
||||
* Returns null if the file doesn't exist or cannot be read.
|
||||
|
|
@ -34,6 +51,79 @@ function readCgroupMemoryValue(cgroupPath: string, filename: string): number | n
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Memory events from memory.events file.
|
||||
*/
|
||||
export interface MemoryEvents {
|
||||
low: number;
|
||||
high: number;
|
||||
max: number;
|
||||
oom: number;
|
||||
oomKill: number;
|
||||
oomGroupKill: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse memory.events file and return event counts.
|
||||
*/
|
||||
export function getMemoryEvents(): MemoryEvents | null {
|
||||
function readEvents(cgroupPath: string): MemoryEvents | null {
|
||||
try {
|
||||
const filePath = path.join(cgroupPath, 'memory.events');
|
||||
if (!fs.existsSync(filePath)) {
|
||||
return null;
|
||||
}
|
||||
const content = fs.readFileSync(filePath, 'utf-8').trim();
|
||||
const lines = content.split('\n');
|
||||
const events: MemoryEvents = {
|
||||
low: 0,
|
||||
high: 0,
|
||||
max: 0,
|
||||
oom: 0,
|
||||
oomKill: 0,
|
||||
oomGroupKill: 0,
|
||||
};
|
||||
|
||||
for (const line of lines) {
|
||||
const [key, value] = line.split(' ');
|
||||
if (!key || !value) continue;
|
||||
const numValue = parseInt(value, 10);
|
||||
if (isNaN(numValue)) continue;
|
||||
|
||||
switch (key) {
|
||||
case 'low':
|
||||
events.low = numValue;
|
||||
break;
|
||||
case 'high':
|
||||
events.high = numValue;
|
||||
break;
|
||||
case 'max':
|
||||
events.max = numValue;
|
||||
break;
|
||||
case 'oom':
|
||||
events.oom = numValue;
|
||||
break;
|
||||
case 'oom_kill':
|
||||
events.oomKill = numValue;
|
||||
break;
|
||||
case 'oom_group_kill':
|
||||
events.oomGroupKill = numValue;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return events;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
let events = readEvents(SYSTEM_CGROUP_PATH);
|
||||
if (events === null) {
|
||||
events = readEvents(FALLBACK_CGROUP_PATH);
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the system memory limit from the cgroup.
|
||||
*/
|
||||
|
|
@ -168,6 +258,43 @@ export interface SystemMemoryStatus {
|
|||
underPressure: boolean;
|
||||
/** OOM risk level */
|
||||
oomRisk: 'none' | 'low' | 'medium' | 'high' | 'critical';
|
||||
/** OOM kill counter from memory.events */
|
||||
oomKill: number;
|
||||
/** Total OOM events count */
|
||||
oom: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a memory sample to the history buffer (circular buffer).
|
||||
*/
|
||||
export function addMemoryHistorySample(usage: number, usagePercent: number, swapUsage: number | null): void {
|
||||
const sample: MemoryHistorySample = {
|
||||
timestamp: Date.now(),
|
||||
usage,
|
||||
usagePercent,
|
||||
swapUsage,
|
||||
};
|
||||
|
||||
// Initialize or replace in circular buffer
|
||||
if (memoryHistory.length < MAX_HISTORY_SAMPLES) {
|
||||
memoryHistory.push(sample);
|
||||
} else {
|
||||
memoryHistory[historyIndex] = sample;
|
||||
historyIndex = (historyIndex + 1) % MAX_HISTORY_SAMPLES;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all memory history samples.
|
||||
* Returns samples in chronological order (oldest to newest).
|
||||
*/
|
||||
export function getMemoryHistory(): MemoryHistorySample[] {
|
||||
if (memoryHistory.length < MAX_HISTORY_SAMPLES) {
|
||||
// Buffer not full, return as-is
|
||||
return [...memoryHistory];
|
||||
}
|
||||
// Buffer full, return from historyIndex to end, then start to historyIndex
|
||||
return [...memoryHistory.slice(historyIndex), ...memoryHistory.slice(0, historyIndex)];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -182,6 +309,7 @@ export function getSystemMemoryStatus(): SystemMemoryStatus {
|
|||
const cgroupSwapUsage = getSystemSwapUsage();
|
||||
const swapInfo = getSwapInfo();
|
||||
const fabricRss = getFabricRss();
|
||||
const memoryEvents = getMemoryEvents();
|
||||
|
||||
// Calculate cgroup usage percentage
|
||||
let cgroupUsagePercent: number | null = null;
|
||||
|
|
@ -198,6 +326,11 @@ export function getSystemMemoryStatus(): SystemMemoryStatus {
|
|||
underPressure = cgroupUsagePercent > 90;
|
||||
}
|
||||
|
||||
// Add to history buffer for sparkline
|
||||
if (cgroupUsage !== null && cgroupUsagePercent !== null) {
|
||||
addMemoryHistorySample(cgroupUsage, cgroupUsagePercent, cgroupSwapUsage);
|
||||
}
|
||||
|
||||
// Determine OOM risk level
|
||||
let oomRisk: 'none' | 'low' | 'medium' | 'high' | 'critical' = 'none';
|
||||
if (cgroupUsagePercent !== null) {
|
||||
|
|
@ -234,6 +367,8 @@ export function getSystemMemoryStatus(): SystemMemoryStatus {
|
|||
cgroupUsagePercent,
|
||||
underPressure,
|
||||
oomRisk,
|
||||
oomKill: memoryEvents?.oomKill ?? 0,
|
||||
oom: memoryEvents?.oom ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ import FleetSummaryBar from './components/FleetSummaryBar';
|
|||
import HistoricalSessionsPanel from './components/HistoricalSessionsPanel';
|
||||
import WorkerAnalyticsPanel from './components/WorkerAnalyticsPanel';
|
||||
import CommandPalette from './components/CommandPalette';
|
||||
import SystemMemoryPanel from './components/SystemMemoryPanel';
|
||||
import { SystemMemoryPanel } from './components/SystemMemoryPanel';
|
||||
import { Agentation } from 'agentation';
|
||||
import { extractReplayFromUrl, ReplayExport } from './utils/replayExport';
|
||||
import { FocusPresetManager, createWebPresetManager, FocusPreset } from './utils/focusPresets';
|
||||
|
|
@ -554,6 +554,8 @@ const App: React.FC = () => {
|
|||
setShowHistoricalSessions(true);
|
||||
} else if (action === 'show:worker-analytics') {
|
||||
setShowWorkerAnalytics(true);
|
||||
} else if (action === 'show:memory') {
|
||||
setShowSystemMemory(true);
|
||||
} else if (action.startsWith('worker:')) {
|
||||
const workerId = action.slice('worker:'.length);
|
||||
setSelectedWorker(workerId);
|
||||
|
|
@ -1111,6 +1113,13 @@ const App: React.FC = () => {
|
|||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{showSystemMemory && (
|
||||
<SystemMemoryPanel
|
||||
visible={showSystemMemory}
|
||||
onClose={() => setShowSystemMemory(false)}
|
||||
/>
|
||||
)}
|
||||
</main>
|
||||
|
||||
<CommandPalette
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ const DEFAULT_COMMANDS: CommandSuggestion[] = [
|
|||
{ id: 'show-budget', label: 'Show budget alerts', category: 'Commands', action: 'show:budget', icon: '%' },
|
||||
{ id: 'show-errors', label: 'Show error groups', category: 'Commands', action: 'show:errors', icon: '🐛' },
|
||||
{ id: 'show-narrative', label: 'Show semantic narrative', category: 'Commands', action: 'show:narrative', icon: '📝' },
|
||||
{ id: 'show-memory', label: 'Show system memory panel', category: 'Commands', action: 'show:memory', icon: '💾' },
|
||||
// Focus mode
|
||||
{ id: 'focus-toggle', label: 'Toggle focus mode', category: 'Commands', action: 'focus:toggle', icon: '📌' },
|
||||
{ id: 'focus-clear', label: 'Clear pinned items', category: 'Commands', action: 'focus:clear', icon: '📍' },
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ interface SystemMemoryStatus {
|
|||
cgroupUsagePercent: number | null;
|
||||
underPressure: boolean;
|
||||
oomRisk: 'none' | 'low' | 'medium' | 'high' | 'critical';
|
||||
oomKill: number;
|
||||
oom: number;
|
||||
}
|
||||
|
||||
interface FormattedMemory {
|
||||
|
|
@ -37,6 +39,21 @@ interface OomAlert {
|
|||
timestamp: number;
|
||||
}
|
||||
|
||||
interface MemoryHistorySample {
|
||||
timestamp: number;
|
||||
usage: number;
|
||||
usagePercent: number;
|
||||
swapUsage: number | null;
|
||||
formattedUsage: string;
|
||||
formattedSwapUsage: string;
|
||||
}
|
||||
|
||||
interface MemoryHistoryResponse {
|
||||
samples: MemoryHistorySample[];
|
||||
count: number;
|
||||
maxSamples: number;
|
||||
}
|
||||
|
||||
interface SystemMemoryPanelProps {
|
||||
visible: boolean;
|
||||
onClose: () => void;
|
||||
|
|
@ -74,6 +91,7 @@ export const SystemMemoryPanel: React.FC<SystemMemoryPanelProps> = ({ visible, o
|
|||
const [memoryStatus, setMemoryStatus] = useState<SystemMemoryStatus | null>(null);
|
||||
const [formattedMemory, setFormattedMemory] = useState<FormattedMemory | null>(null);
|
||||
const [oomAlert, setOomAlert] = useState<OomAlert | null>(null);
|
||||
const [memoryHistory, setMemoryHistory] = useState<MemoryHistorySample[]>([]);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
|
|
@ -108,17 +126,32 @@ export const SystemMemoryPanel: React.FC<SystemMemoryPanelProps> = ({ visible, o
|
|||
}
|
||||
}, []);
|
||||
|
||||
const fetchMemoryHistory = useCallback(async () => {
|
||||
try {
|
||||
const response = await fetch('/api/system/memory/history');
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
const data: MemoryHistoryResponse = await response.json();
|
||||
setMemoryHistory(data.samples);
|
||||
} catch (err) {
|
||||
console.error('Failed to fetch memory history:', err);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (visible) {
|
||||
fetchSystemMemory();
|
||||
fetchOomAlert();
|
||||
fetchMemoryHistory();
|
||||
const interval = setInterval(() => {
|
||||
fetchSystemMemory();
|
||||
fetchOomAlert();
|
||||
fetchMemoryHistory();
|
||||
}, 5000);
|
||||
return () => clearInterval(interval);
|
||||
}
|
||||
}, [visible, fetchSystemMemory, fetchOomAlert]);
|
||||
}, [visible, fetchSystemMemory, fetchOomAlert, fetchMemoryHistory]);
|
||||
|
||||
if (!visible) return null;
|
||||
|
||||
|
|
@ -163,6 +196,58 @@ export const SystemMemoryPanel: React.FC<SystemMemoryPanelProps> = ({ visible, o
|
|||
{/* Cgroup Memory Section */}
|
||||
<div className="memory-section">
|
||||
<h3>Cgroup Memory</h3>
|
||||
|
||||
{/* OOM Kill Counter */}
|
||||
{(memoryStatus.oomKill > 0 || memoryStatus.oom > 0) && (
|
||||
<div className="oom-kill-counter">
|
||||
<span className="oom-kill-icon">💀</span>
|
||||
<span className="oom-kill-text">
|
||||
{memoryStatus.oomKill > 0 && `${memoryStatus.oomKill} OOM kill${memoryStatus.oomKill > 1 ? 's' : ''}`}
|
||||
{memoryStatus.oom > 0 && memoryStatus.oomKill !== memoryStatus.oom && ` · ${memoryStatus.oom} OOM event${memoryStatus.oom > 1 ? 's' : ''}`}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* 5-minute sparkline */}
|
||||
{memoryHistory.length > 0 && (
|
||||
<div className="memory-sparkline-container">
|
||||
<div className="sparkline-label">5-minute trend</div>
|
||||
<div className="memory-sparkline">
|
||||
{memoryHistory.map((sample, index) => {
|
||||
const maxPercent = Math.max(...memoryHistory.map(s => s.usagePercent), 1);
|
||||
const heightPercent = (sample.usagePercent / maxPercent) * 100;
|
||||
const getColor = (pct: number) => {
|
||||
if (pct >= 98) return '#d32f2f';
|
||||
if (pct >= 95) return '#f44336';
|
||||
if (pct >= 90) return '#ff5722';
|
||||
if (pct >= 80) return '#ff9800';
|
||||
return '#4caf50';
|
||||
};
|
||||
return (
|
||||
<div
|
||||
key={sample.timestamp}
|
||||
className="sparkline-bar"
|
||||
style={{
|
||||
height: `${Math.max(5, heightPercent)}%`,
|
||||
backgroundColor: getColor(sample.usagePercent),
|
||||
}}
|
||||
title={`${new Date(sample.timestamp).toLocaleTimeString()}: ${sample.formattedUsage} (${sample.usagePercent.toFixed(1)}%)`}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
<div className="sparkline-legend">
|
||||
<span className="legend-dot" style={{ backgroundColor: '#4caf50' }}></span>
|
||||
<span><80%</span>
|
||||
<span className="legend-dot" style={{ backgroundColor: '#ff9800' }}></span>
|
||||
<span>80-90%</span>
|
||||
<span className="legend-dot" style={{ backgroundColor: '#ff5722' }}></span>
|
||||
<span>90-95%</span>
|
||||
<span className="legend-dot" style={{ backgroundColor: '#f44336' }}></span>
|
||||
<span>≥95%</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<div className="memory-bar-container">
|
||||
<div className="memory-bar-label">
|
||||
<span>Usage</span>
|
||||
|
|
@ -538,6 +623,80 @@ export const SystemMemoryPanel: React.FC<SystemMemoryPanelProps> = ({ visible, o
|
|||
.system-memory-panel .refresh-button:hover {
|
||||
background: var(--bg-secondary);
|
||||
}
|
||||
|
||||
/* OOM Kill Counter */
|
||||
.system-memory-panel .oom-kill-counter {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 10px 12px;
|
||||
background: rgba(211, 47, 47, 0.1);
|
||||
border: 1px solid rgba(211, 47, 47, 0.3);
|
||||
border-radius: 6px;
|
||||
margin-bottom: 12px;
|
||||
color: #d32f2f;
|
||||
font-weight: 600;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.system-memory-panel .oom-kill-icon {
|
||||
font-size: 18px;
|
||||
}
|
||||
|
||||
.system-memory-panel .oom-kill-text {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
/* Memory Sparkline */
|
||||
.system-memory-panel .memory-sparkline-container {
|
||||
margin-bottom: 12px;
|
||||
padding: 12px;
|
||||
background: var(--bg-secondary);
|
||||
border-radius: 6px;
|
||||
}
|
||||
|
||||
.system-memory-panel .sparkline-label {
|
||||
font-size: 12px;
|
||||
color: var(--text-secondary);
|
||||
margin-bottom: 8px;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.system-memory-panel .memory-sparkline {
|
||||
display: flex;
|
||||
align-items: flex-end;
|
||||
gap: 2px;
|
||||
height: 60px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.system-memory-panel .sparkline-bar {
|
||||
flex: 1;
|
||||
min-width: 2px;
|
||||
max-width: 12px;
|
||||
border-radius: 2px 2px 0 0;
|
||||
transition: height 0.3s ease, background-color 0.3s ease;
|
||||
cursor: crosshair;
|
||||
}
|
||||
|
||||
.system-memory-panel .sparkline-bar:hover {
|
||||
opacity: 0.8;
|
||||
}
|
||||
|
||||
.system-memory-panel .sparkline-legend {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
font-size: 10px;
|
||||
color: var(--text-secondary);
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.system-memory-panel .legend-dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
}
|
||||
`}</style>
|
||||
</div>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1637,6 +1637,25 @@ export function createWebServer(options: WebServerOptions): WebServer {
|
|||
});
|
||||
});
|
||||
|
||||
// Get memory history for sparkline (last 5 minutes @ 10s intervals = 30 samples)
|
||||
app.get('/api/system/memory/history', async (_req: Request, res: Response) => {
|
||||
const { getMemoryHistory, formatBytes } = await import('../systemCgroupMonitor.js');
|
||||
const history = getMemoryHistory();
|
||||
|
||||
res.json({
|
||||
samples: history.map(sample => ({
|
||||
timestamp: sample.timestamp,
|
||||
usage: sample.usage,
|
||||
usagePercent: sample.usagePercent,
|
||||
swapUsage: sample.swapUsage,
|
||||
formattedUsage: formatBytes(sample.usage),
|
||||
formattedSwapUsage: formatBytes(sample.swapUsage),
|
||||
})),
|
||||
count: history.length,
|
||||
maxSamples: 30, // 5 minutes @ 10s intervals
|
||||
});
|
||||
});
|
||||
|
||||
// Get human-readable memory summary
|
||||
app.get('/api/system/memory/summary', async (_req: Request, res: Response) => {
|
||||
const { getMemorySummary } = await import('../systemCgroupMonitor.js');
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue