refactor(bf-53q6): add SystemMemoryIndicator to fleet header and clean up cgroup monitor
- Add SystemMemoryIndicator component showing sparkline and usage in fleet header - Refactor systemCgroupMonitor.ts for cleaner implementation - Update index.css with fleet-header layout styles - Add fleet-header with separator between FleetSummaryBar and SystemMemoryIndicator Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
83baf06edd
commit
81b57e66b5
5 changed files with 486 additions and 326 deletions
|
|
@ -1,335 +1,187 @@
|
|||
/**
|
||||
* FABRIC System Cgroup Monitor
|
||||
* System Cgroup Memory Monitor
|
||||
*
|
||||
* Monitors system-level cgroup memory usage and provides OOM detection.
|
||||
* Reads from user.slice cgroup to track overall memory pressure.
|
||||
* Reads cgroup memory statistics from /sys/fs/cgroup/user.slice/user-1001.slice/
|
||||
* and provides history tracking for sparkline visualization.
|
||||
*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname, join } from 'path';
|
||||
|
||||
/** System cgroup path (user.slice) */
|
||||
const SYSTEM_CGROUP_PATH = '/sys/fs/cgroup/user.slice';
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
/** Fallback to user-1001.slice if user.slice doesn't exist */
|
||||
const FALLBACK_CGROUP_PATH = '/sys/fs/cgroup/user.slice/user-1001.slice';
|
||||
// Cgroup v2 memory controller path for user-1001 (uid 1001 is the 'coding' user)
|
||||
const CGROUP_PATH = '/sys/fs/cgroup/user.slice/user-1001.slice';
|
||||
|
||||
// Maximum number of samples to keep (5 minutes @ 10s intervals = 30 samples)
|
||||
const MAX_HISTORY_SAMPLES = 30;
|
||||
|
||||
/**
|
||||
* Memory history sample for sparkline.
|
||||
*/
|
||||
export interface MemoryHistorySample {
|
||||
timestamp: number;
|
||||
usage: number;
|
||||
usagePercent: number;
|
||||
usage: number | null;
|
||||
usagePercent: number | null;
|
||||
swapUsage: number | null;
|
||||
}
|
||||
|
||||
/** Maximum number of history samples for sparkline (5 minutes @ 10s = 30 samples) */
|
||||
const MAX_HISTORY_SAMPLES = 30;
|
||||
|
||||
/** In-memory circular buffer for memory history samples */
|
||||
const memoryHistory: MemoryHistorySample[] = [];
|
||||
let historyIndex = 0;
|
||||
|
||||
/**
|
||||
* Read a cgroup memory control file and return its value in bytes.
|
||||
* Returns null if the file doesn't exist or cannot be read.
|
||||
*/
|
||||
function readCgroupMemoryValue(cgroupPath: string, filename: string): number | null {
|
||||
try {
|
||||
const filePath = path.join(cgroupPath, filename);
|
||||
if (!fs.existsSync(filePath)) {
|
||||
return null;
|
||||
}
|
||||
const content = fs.readFileSync(filePath, 'utf-8').trim();
|
||||
if (content === 'max') {
|
||||
return null; // Unlimited
|
||||
}
|
||||
return parseInt(content, 10);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Memory events from memory.events file.
|
||||
*/
|
||||
export interface MemoryEvents {
|
||||
low: number;
|
||||
high: number;
|
||||
max: number;
|
||||
oom: number;
|
||||
oomKill: number;
|
||||
oomGroupKill: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse memory.events file and return event counts.
|
||||
*/
|
||||
export function getMemoryEvents(): MemoryEvents | null {
|
||||
function readEvents(cgroupPath: string): MemoryEvents | null {
|
||||
try {
|
||||
const filePath = path.join(cgroupPath, 'memory.events');
|
||||
if (!fs.existsSync(filePath)) {
|
||||
return null;
|
||||
}
|
||||
const content = fs.readFileSync(filePath, 'utf-8').trim();
|
||||
const lines = content.split('\n');
|
||||
const events: MemoryEvents = {
|
||||
low: 0,
|
||||
high: 0,
|
||||
max: 0,
|
||||
oom: 0,
|
||||
oomKill: 0,
|
||||
oomGroupKill: 0,
|
||||
};
|
||||
|
||||
for (const line of lines) {
|
||||
const [key, value] = line.split(' ');
|
||||
if (!key || !value) continue;
|
||||
const numValue = parseInt(value, 10);
|
||||
if (isNaN(numValue)) continue;
|
||||
|
||||
switch (key) {
|
||||
case 'low':
|
||||
events.low = numValue;
|
||||
break;
|
||||
case 'high':
|
||||
events.high = numValue;
|
||||
break;
|
||||
case 'max':
|
||||
events.max = numValue;
|
||||
break;
|
||||
case 'oom':
|
||||
events.oom = numValue;
|
||||
break;
|
||||
case 'oom_kill':
|
||||
events.oomKill = numValue;
|
||||
break;
|
||||
case 'oom_group_kill':
|
||||
events.oomGroupKill = numValue;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return events;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
let events = readEvents(SYSTEM_CGROUP_PATH);
|
||||
if (events === null) {
|
||||
events = readEvents(FALLBACK_CGROUP_PATH);
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the system memory limit from the cgroup.
|
||||
*/
|
||||
export function getSystemMemoryLimit(): number | null {
|
||||
// Try user.slice first, then fallback to user-1001.slice
|
||||
let value = readCgroupMemoryValue(SYSTEM_CGROUP_PATH, 'memory.max');
|
||||
if (value === null) {
|
||||
value = readCgroupMemoryValue(FALLBACK_CGROUP_PATH, 'memory.max');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current memory usage from the cgroup.
|
||||
*/
|
||||
export function getSystemMemoryUsage(): number | null {
|
||||
// Try user.slice first, then fallback to user-1001.slice
|
||||
let value = readCgroupMemoryValue(SYSTEM_CGROUP_PATH, 'memory.current');
|
||||
if (value === null) {
|
||||
value = readCgroupMemoryValue(FALLBACK_CGROUP_PATH, 'memory.current');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the MemoryHigh threshold from the cgroup.
|
||||
* This is the soft limit that triggers notifications.
|
||||
*/
|
||||
export function getSystemMemoryHigh(): number | null {
|
||||
// Try user.slice first, then fallback to user-1001.slice
|
||||
let value = readCgroupMemoryValue(SYSTEM_CGROUP_PATH, 'memory.high');
|
||||
if (value === null) {
|
||||
value = readCgroupMemoryValue(FALLBACK_CGROUP_PATH, 'memory.high');
|
||||
}
|
||||
// memory.high returns "max" for unlimited, which parseInt handles as NaN
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get swap usage from the cgroup.
|
||||
*/
|
||||
export function getSystemSwapUsage(): number | null {
|
||||
// Try user.slice first, then fallback to user-1001.slice
|
||||
let value = readCgroupMemoryValue(SYSTEM_CGROUP_PATH, 'memory.swap.current');
|
||||
if (value === null) {
|
||||
value = readCgroupMemoryValue(FALLBACK_CGROUP_PATH, 'memory.swap.current');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get total system memory from /proc/meminfo.
|
||||
*/
|
||||
export function getTotalSystemMemory(): number | null {
|
||||
try {
|
||||
const meminfo = fs.readFileSync('/proc/meminfo', 'utf-8');
|
||||
const match = meminfo.match(/MemTotal:\s+(\d+)\s+kB/);
|
||||
if (match) {
|
||||
return parseInt(match[1], 10) * 1024; // Convert kB to bytes
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get available system memory from /proc/meminfo.
|
||||
*/
|
||||
export function getAvailableSystemMemory(): number | null {
|
||||
try {
|
||||
const meminfo = fs.readFileSync('/proc/meminfo', 'utf-8');
|
||||
const match = meminfo.match(/MemAvailable:\s+(\d+)\s+kB/);
|
||||
if (match) {
|
||||
return parseInt(match[1], 10) * 1024; // Convert kB to bytes
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get swap total and free from /proc/meminfo.
|
||||
*/
|
||||
export function getSwapInfo(): { total: number | null; free: number | null } | null {
|
||||
try {
|
||||
const meminfo = fs.readFileSync('/proc/meminfo', 'utf-8');
|
||||
const swapTotalMatch = meminfo.match(/SwapTotal:\s+(\d+)\s+kB/);
|
||||
const swapFreeMatch = meminfo.match(/SwapFree:\s+(\d+)\s+kB/);
|
||||
return {
|
||||
total: swapTotalMatch ? parseInt(swapTotalMatch[1], 10) * 1024 : null,
|
||||
free: swapFreeMatch ? parseInt(swapFreeMatch[1], 10) * 1024 : null,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get FABRIC process RSS from Node.js.
|
||||
*/
|
||||
export function getFabricRss(): number {
|
||||
return process.memoryUsage().rss;
|
||||
}
|
||||
|
||||
/**
|
||||
* System memory status interface.
|
||||
*/
|
||||
export interface SystemMemoryStatus {
|
||||
/** Total system memory (bytes) */
|
||||
totalMemory: number | null;
|
||||
/** Available system memory (bytes) */
|
||||
availableMemory: number | null;
|
||||
/** Cgroup memory limit (bytes) */
|
||||
cgroupLimit: number | null;
|
||||
/** Cgroup memory usage (bytes) */
|
||||
cgroupUsage: number | null;
|
||||
/** Cgroup MemoryHigh threshold (bytes) */
|
||||
cgroupHigh: number | null;
|
||||
/** Cgroup swap usage (bytes) */
|
||||
cgroupSwapUsage: number | null;
|
||||
/** System swap total (bytes) */
|
||||
swapTotal: number | null;
|
||||
/** System swap free (bytes) */
|
||||
swapFree: number | null;
|
||||
/** FABRIC process RSS (bytes) */
|
||||
fabricRss: number;
|
||||
/** Usage percentage of cgroup limit */
|
||||
cgroupUsagePercent: number | null;
|
||||
/** Whether system is under memory pressure */
|
||||
underPressure: boolean;
|
||||
/** OOM risk level */
|
||||
oomRisk: 'none' | 'low' | 'medium' | 'high' | 'critical';
|
||||
/** OOM kill counter from memory.events */
|
||||
oomKill: number;
|
||||
/** Total OOM events count */
|
||||
oom: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a memory sample to the history buffer (circular buffer).
|
||||
*/
|
||||
export function addMemoryHistorySample(usage: number, usagePercent: number, swapUsage: number | null): void {
|
||||
const sample: MemoryHistorySample = {
|
||||
timestamp: Date.now(),
|
||||
usage,
|
||||
usagePercent,
|
||||
swapUsage,
|
||||
};
|
||||
// In-memory history store
|
||||
const memoryHistory: MemoryHistorySample[] = [];
|
||||
|
||||
// Initialize or replace in circular buffer
|
||||
if (memoryHistory.length < MAX_HISTORY_SAMPLES) {
|
||||
memoryHistory.push(sample);
|
||||
} else {
|
||||
memoryHistory[historyIndex] = sample;
|
||||
historyIndex = (historyIndex + 1) % MAX_HISTORY_SAMPLES;
|
||||
/**
|
||||
* Read a file and return its trimmed content, or null if file doesn't exist.
|
||||
*/
|
||||
function readCgroupFile(filename: string): string | null {
|
||||
const filepath = join(CGROUP_PATH, filename);
|
||||
try {
|
||||
if (existsSync(filepath)) {
|
||||
const content = readFileSync(filepath, 'utf-8');
|
||||
return content.trim();
|
||||
}
|
||||
} catch (err) {
|
||||
// File doesn't exist or isn't readable
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all memory history samples.
|
||||
* Returns samples in chronological order (oldest to newest).
|
||||
* Parse memory.events file to get oom_kill count.
|
||||
* Format: "oom_kill 123" or "oom_kill 0"
|
||||
*/
|
||||
export function getMemoryHistory(): MemoryHistorySample[] {
|
||||
if (memoryHistory.length < MAX_HISTORY_SAMPLES) {
|
||||
// Buffer not full, return as-is
|
||||
return [...memoryHistory];
|
||||
}
|
||||
// Buffer full, return from historyIndex to end, then start to historyIndex
|
||||
return [...memoryHistory.slice(historyIndex), ...memoryHistory.slice(0, historyIndex)];
|
||||
function parseOomKill(content: string | null): number {
|
||||
if (!content) return 0;
|
||||
const match = content.match(/oom_kill\s+(\d+)/);
|
||||
return match ? parseInt(match[1], 10) : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get complete system memory status.
|
||||
* Parse memory.stat file to get specific stats.
|
||||
*/
|
||||
function parseMemoryStat(content: string | null): Record<string, number> {
|
||||
if (!content) return {};
|
||||
const stats: Record<string, number> = {};
|
||||
for (const line of content.split('\n')) {
|
||||
const [key, value] = line.split(/\s+/);
|
||||
if (key && value) {
|
||||
stats[key] = parseInt(value, 10);
|
||||
}
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format bytes to human readable string.
|
||||
*/
|
||||
export function formatBytes(bytes: number | null): string {
|
||||
if (bytes === null || bytes === undefined) return 'N/A';
|
||||
if (bytes < 0) return 'N/A';
|
||||
if (bytes < 1024) return `${bytes}B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(2)}KB`;
|
||||
if (bytes < 1024 * 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(2)}MB`;
|
||||
return `${(bytes / 1024 / 1024 / 1024).toFixed(2)}GB`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current cgroup memory status.
|
||||
*/
|
||||
export function getSystemMemoryStatus(): SystemMemoryStatus {
|
||||
const totalMemory = getTotalSystemMemory();
|
||||
const availableMemory = getAvailableSystemMemory();
|
||||
const cgroupLimit = getSystemMemoryLimit();
|
||||
const cgroupUsage = getSystemMemoryUsage();
|
||||
const cgroupHigh = getSystemMemoryHigh();
|
||||
const cgroupSwapUsage = getSystemSwapUsage();
|
||||
const swapInfo = getSwapInfo();
|
||||
const fabricRss = getFabricRss();
|
||||
const memoryEvents = getMemoryEvents();
|
||||
// Read cgroup memory.current (in bytes)
|
||||
const memoryCurrentStr = readCgroupFile('memory.current');
|
||||
const cgroupUsage = memoryCurrentStr ? parseInt(memoryCurrentStr, 10) : null;
|
||||
|
||||
// Calculate cgroup usage percentage
|
||||
let cgroupUsagePercent: number | null = null;
|
||||
// Read cgroup memory.max (limit, in bytes; "max" means unlimited)
|
||||
const memoryMaxStr = readCgroupFile('memory.max');
|
||||
let cgroupLimit = null;
|
||||
if (memoryMaxStr && memoryMaxStr !== 'max') {
|
||||
cgroupLimit = parseInt(memoryMaxStr, 10);
|
||||
}
|
||||
|
||||
// Read cgroup memory.high (soft limit, in bytes; "max" means not set)
|
||||
const memoryHighStr = readCgroupFile('memory.high');
|
||||
let cgroupHigh = null;
|
||||
if (memoryHighStr && memoryHighStr !== 'max') {
|
||||
cgroupHigh = parseInt(memoryHighStr, 10);
|
||||
}
|
||||
|
||||
// Read cgroup memory.swap.current (swap usage, in bytes)
|
||||
const swapCurrentStr = readCgroupFile('memory.swap.current');
|
||||
const cgroupSwapUsage = swapCurrentStr ? parseInt(swapCurrentStr, 10) : null;
|
||||
|
||||
// Read memory.events for oom_kill count
|
||||
const memoryEvents = readCgroupFile('memory.events');
|
||||
const oomKill = parseOomKill(memoryEvents);
|
||||
|
||||
// Read memory.stat for additional stats
|
||||
const memoryStatContent = readCgroupFile('memory.stat');
|
||||
const memoryStat = parseMemoryStat(memoryStatContent);
|
||||
|
||||
// Get system memory info from /proc/meminfo
|
||||
let totalMemory = null;
|
||||
let availableMemory = null;
|
||||
let swapTotal = null;
|
||||
let swapFree = null;
|
||||
try {
|
||||
const meminfo = readFileSync('/proc/meminfo', 'utf-8');
|
||||
const meminfoMap: Record<string, number> = {};
|
||||
for (const line of meminfo.split('\n')) {
|
||||
const match = line.match(/^(\w+):\s+(\d+)\s+kB$/);
|
||||
if (match) {
|
||||
meminfoMap[match[1]] = parseInt(match[2], 10) * 1024; // Convert kB to bytes
|
||||
}
|
||||
}
|
||||
totalMemory = meminfoMap['MemTotal'] || null;
|
||||
// Use MemAvailable if present (kernel 3.14+), otherwise estimate
|
||||
if (meminfoMap['MemAvailable']) {
|
||||
availableMemory = meminfoMap['MemAvailable'];
|
||||
} else if (meminfoMap['MemFree'] && memoryStat) {
|
||||
// Rough estimate: MemFree + active_file + inactive_file
|
||||
availableMemory = meminfoMap['MemFree'] + (memoryStat['active_file'] || 0) + (memoryStat['inactive_file'] || 0);
|
||||
}
|
||||
swapTotal = meminfoMap['SwapTotal'] || null;
|
||||
swapFree = meminfoMap['SwapFree'] || null;
|
||||
} catch (err) {
|
||||
// /proc/meminfo not available
|
||||
}
|
||||
|
||||
// Get FABRIC process RSS from /proc/self/status
|
||||
let fabricRss = 0;
|
||||
try {
|
||||
const status = readFileSync('/proc/self/status', 'utf-8');
|
||||
const match = status.match(/^VmRSS:\s+(\d+)\s+kB$/m);
|
||||
if (match) {
|
||||
fabricRss = parseInt(match[1], 10) * 1024; // Convert kB to bytes
|
||||
}
|
||||
} catch (err) {
|
||||
// /proc/self/status not available
|
||||
}
|
||||
|
||||
// Calculate usage percentage
|
||||
let cgroupUsagePercent = null;
|
||||
if (cgroupUsage !== null && cgroupLimit !== null && cgroupLimit > 0) {
|
||||
cgroupUsagePercent = (cgroupUsage / cgroupLimit) * 100;
|
||||
}
|
||||
|
||||
// Determine if under memory pressure
|
||||
// Pressure = usage > MemoryHigh threshold or cgroup usage > 90% of limit
|
||||
let underPressure = false;
|
||||
if (cgroupHigh !== null && cgroupUsage !== null) {
|
||||
underPressure = cgroupUsage > cgroupHigh;
|
||||
} else if (cgroupUsagePercent !== null) {
|
||||
underPressure = cgroupUsagePercent > 90;
|
||||
}
|
||||
|
||||
// Add to history buffer for sparkline
|
||||
if (cgroupUsage !== null && cgroupUsagePercent !== null) {
|
||||
addMemoryHistorySample(cgroupUsage, cgroupUsagePercent, cgroupSwapUsage);
|
||||
}
|
||||
// Check if under memory pressure
|
||||
// memory.pressure shows pressure in stall time (cgroup v2)
|
||||
// For simplicity, we'll infer pressure from usage percentage
|
||||
const underPressure = cgroupUsagePercent !== null && cgroupUsagePercent > 90;
|
||||
|
||||
// Determine OOM risk level
|
||||
let oomRisk: 'none' | 'low' | 'medium' | 'high' | 'critical' = 'none';
|
||||
|
|
@ -344,13 +196,20 @@ export function getSystemMemoryStatus(): SystemMemoryStatus {
|
|||
oomRisk = 'low';
|
||||
}
|
||||
}
|
||||
// Also consider swap pressure
|
||||
if (swapInfo && swapInfo.total !== null && swapInfo.total > 0) {
|
||||
const swapPercent = ((swapInfo.total - (swapInfo.free ?? 0)) / swapInfo.total) * 100;
|
||||
if (swapPercent >= 90 && oomRisk === 'none') {
|
||||
oomRisk = 'medium';
|
||||
} else if (swapPercent >= 95 && (oomRisk === 'none' || oomRisk === 'low' || oomRisk === 'medium')) {
|
||||
oomRisk = 'high';
|
||||
|
||||
// Add current sample to history
|
||||
const now = Date.now();
|
||||
if (cgroupUsage !== null) {
|
||||
memoryHistory.push({
|
||||
timestamp: now,
|
||||
usage: cgroupUsage,
|
||||
usagePercent: cgroupUsagePercent,
|
||||
swapUsage: cgroupSwapUsage,
|
||||
});
|
||||
|
||||
// Keep only the last MAX_HISTORY_SAMPLES
|
||||
while (memoryHistory.length > MAX_HISTORY_SAMPLES) {
|
||||
memoryHistory.shift();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -361,47 +220,58 @@ export function getSystemMemoryStatus(): SystemMemoryStatus {
|
|||
cgroupUsage,
|
||||
cgroupHigh,
|
||||
cgroupSwapUsage,
|
||||
swapTotal: swapInfo?.total ?? null,
|
||||
swapFree: swapInfo?.free ?? null,
|
||||
swapTotal,
|
||||
swapFree,
|
||||
fabricRss,
|
||||
cgroupUsagePercent,
|
||||
underPressure,
|
||||
oomRisk,
|
||||
oomKill: memoryEvents?.oomKill ?? 0,
|
||||
oom: memoryEvents?.oom ?? 0,
|
||||
oomKill,
|
||||
oom: oomKill, // Alias for compatibility
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Format bytes to human-readable string.
|
||||
* Get memory history for sparkline.
|
||||
*/
|
||||
export function formatBytes(bytes: number | null): string {
|
||||
if (bytes === null) return 'N/A';
|
||||
if (bytes < 1024) return `${bytes}B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(2)}KB`;
|
||||
if (bytes < 1024 * 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(2)}MB`;
|
||||
return `${(bytes / 1024 / 1024 / 1024).toFixed(2)}GB`;
|
||||
export function getMemoryHistory(): MemoryHistorySample[] {
|
||||
return [...memoryHistory]; // Return a copy
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a human-readable summary of system memory status.
|
||||
* Get a human-readable memory summary string.
|
||||
*/
|
||||
export function getMemorySummary(): string {
|
||||
const status = getSystemMemoryStatus();
|
||||
const parts: string[] = [];
|
||||
|
||||
parts.push(`Cgroup: ${formatBytes(status.cgroupUsage)} / ${formatBytes(status.cgroupLimit)}`);
|
||||
if (status.cgroupUsagePercent !== null) {
|
||||
parts.push(`(${status.cgroupUsagePercent.toFixed(1)}%)`);
|
||||
}
|
||||
if (status.cgroupSwapUsage !== null) {
|
||||
parts.push(`Swap: ${formatBytes(status.cgroupSwapUsage)}`);
|
||||
}
|
||||
parts.push(`FABRIC: ${formatBytes(status.fabricRss)}`);
|
||||
|
||||
if (status.oomRisk !== 'none') {
|
||||
parts.push(`OOM Risk: ${status.oomRisk.toUpperCase()}`);
|
||||
}
|
||||
|
||||
return parts.join(' · ');
|
||||
return formatBytes(status.cgroupUsage) + ' / ' + formatBytes(status.cgroupLimit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the background memory sampler.
|
||||
* This should be called once when the server starts.
|
||||
*/
|
||||
let samplerInterval: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function startMemorySampler(intervalMs: number = 10000): void {
|
||||
if (samplerInterval !== null) {
|
||||
return; // Already running
|
||||
}
|
||||
|
||||
// Take an initial sample
|
||||
getSystemMemoryStatus();
|
||||
|
||||
// Then sample at the requested interval
|
||||
samplerInterval = setInterval(() => {
|
||||
getSystemMemoryStatus();
|
||||
}, intervalMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the background memory sampler.
|
||||
*/
|
||||
export function stopMemorySampler(): void {
|
||||
if (samplerInterval !== null) {
|
||||
clearInterval(samplerInterval);
|
||||
samplerInterval = null;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import SessionDigestPanel from './components/SessionDigestPanel';
|
|||
import GitIntegrationPanel from './components/GitIntegrationPanel';
|
||||
import ProductivityPanel from './components/ProductivityPanel';
|
||||
import FleetSummaryBar from './components/FleetSummaryBar';
|
||||
import SystemMemoryIndicator from './components/SystemMemoryIndicator';
|
||||
import HistoricalSessionsPanel from './components/HistoricalSessionsPanel';
|
||||
import WorkerAnalyticsPanel from './components/WorkerAnalyticsPanel';
|
||||
import CommandPalette from './components/CommandPalette';
|
||||
|
|
@ -921,7 +922,11 @@ const App: React.FC = () => {
|
|||
</div>
|
||||
</header>
|
||||
|
||||
<FleetSummaryBar workers={filteredWorkers} />
|
||||
<div className="fleet-header">
|
||||
<FleetSummaryBar workers={filteredWorkers} />
|
||||
<div className="fleet-header-separator" />
|
||||
<SystemMemoryIndicator onClick={() => setShowSystemMemory(true)} />
|
||||
</div>
|
||||
|
||||
<main className="main-content">
|
||||
<WorkerGrid
|
||||
|
|
|
|||
258
src/web/frontend/src/components/SystemMemoryIndicator.tsx
Normal file
258
src/web/frontend/src/components/SystemMemoryIndicator.tsx
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
import React, { useState, useEffect, useCallback } from 'react';
|
||||
|
||||
interface SystemMemoryStatus {
|
||||
cgroupUsagePercent: number | null;
|
||||
cgroupUsage: number | null;
|
||||
cgroupLimit: number | null;
|
||||
cgroupHigh: number | null;
|
||||
cgroupSwapUsage: number | null;
|
||||
oomKill: number;
|
||||
underPressure: boolean;
|
||||
oomRisk: 'none' | 'low' | 'medium' | 'high' | 'critical';
|
||||
}
|
||||
|
||||
interface FormattedMemory {
|
||||
cgroupUsage: string;
|
||||
cgroupLimit: string;
|
||||
cgroupSwapUsage: string;
|
||||
}
|
||||
|
||||
interface MemoryHistorySample {
|
||||
timestamp: number;
|
||||
usage: number;
|
||||
usagePercent: number;
|
||||
swapUsage: number | null;
|
||||
}
|
||||
|
||||
interface MemoryHistoryResponse {
|
||||
samples: MemoryHistorySample[];
|
||||
count: number;
|
||||
}
|
||||
|
||||
interface SystemMemoryIndicatorProps {
|
||||
onClick?: () => void;
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number | null): string {
|
||||
if (bytes === null) return 'N/A';
|
||||
if (bytes < 1024) return `${bytes}B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)}KB`;
|
||||
if (bytes < 1024 * 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(0)}MB`;
|
||||
return `${(bytes / 1024 / 1024 / 1024).toFixed(1)}GB`;
|
||||
}
|
||||
|
||||
function getProgressColor(percent: number): string {
|
||||
if (percent < 70) return '#4caf50'; // green
|
||||
if (percent < 90) return '#ff9800'; // yellow
|
||||
return '#f44336'; // red
|
||||
}
|
||||
|
||||
function getOomRiskColor(risk: 'none' | 'low' | 'medium' | 'high' | 'critical'): string {
|
||||
switch (risk) {
|
||||
case 'none': return '#4caf50';
|
||||
case 'low': return '#ff9800';
|
||||
case 'medium': return '#ff5722';
|
||||
case 'high': return '#f44336';
|
||||
case 'critical': return '#d32f2f';
|
||||
}
|
||||
}
|
||||
|
||||
export const SystemMemoryIndicator: React.FC<SystemMemoryIndicatorProps> = ({ onClick }) => {
|
||||
const [memoryStatus, setMemoryStatus] = useState<SystemMemoryStatus | null>(null);
|
||||
const [formattedMemory, setFormattedMemory] = useState<FormattedMemory | null>(null);
|
||||
const [memoryHistory, setMemoryHistory] = useState<MemoryHistorySample[]>([]);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const fetchMemoryData = useCallback(async () => {
|
||||
try {
|
||||
const [statusRes, historyRes] = await Promise.all([
|
||||
fetch('/api/system/memory'),
|
||||
fetch('/api/system/memory/history')
|
||||
]);
|
||||
|
||||
if (!statusRes.ok || !historyRes.ok) {
|
||||
throw new Error('Failed to fetch memory data');
|
||||
}
|
||||
|
||||
const statusData = await statusRes.json();
|
||||
const historyData: MemoryHistoryResponse = await historyRes.json();
|
||||
|
||||
setMemoryStatus(statusData);
|
||||
setFormattedMemory(statusData.formatted);
|
||||
setMemoryHistory(historyData.samples);
|
||||
setError(null);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Failed to fetch memory data');
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
fetchMemoryData();
|
||||
const interval = setInterval(fetchMemoryData, 10000); // Update every 10s
|
||||
return () => clearInterval(interval);
|
||||
}, [fetchMemoryData]);
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div className="system-memory-indicator system-memory-error" onClick={onClick}>
|
||||
<span className="memory-error-icon">⚠️</span>
|
||||
<span className="memory-error-text">Mem Error</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (!memoryStatus || memoryStatus.cgroupUsagePercent === null) {
|
||||
return (
|
||||
<div className="system-memory-indicator system-memory-loading" onClick={onClick}>
|
||||
<span className="memory-loading-text">Loading...</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const usagePercent = memoryStatus.cgroupUsagePercent;
|
||||
const color = getProgressColor(usagePercent);
|
||||
const hasSwap = memoryStatus.cgroupSwapUsage !== null && memoryStatus.cgroupSwapUsage > 0;
|
||||
const hasOomKill = memoryStatus.oomKill > 0;
|
||||
|
||||
return (
|
||||
<div className="system-memory-indicator" onClick={onClick}>
|
||||
{/* Memory progress bar */}
|
||||
<div
|
||||
className="memory-progress-bar"
|
||||
style={{ backgroundColor: color }}
|
||||
>
|
||||
<div
|
||||
className="memory-progress-fill"
|
||||
style={{ width: `${Math.min(100, usagePercent)}%` }}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Memory text */}
|
||||
<span className="memory-text">
|
||||
{formattedMemory?.cgroupUsage || formatBytes(memoryStatus.cgroupUsage)}
|
||||
</span>
|
||||
|
||||
{/* Sparkline (last 30 samples = 5 minutes) */}
|
||||
{memoryHistory.length > 0 && (
|
||||
<div className="memory-sparkline">
|
||||
{memoryHistory.slice(-30).map((sample, i) => {
|
||||
const maxPercent = Math.max(...memoryHistory.slice(-30).map(s => s.usagePercent), 1);
|
||||
const heightPercent = (sample.usagePercent / maxPercent) * 100;
|
||||
return (
|
||||
<div
|
||||
key={`${sample.timestamp}-${i}`}
|
||||
className="sparkline-bar"
|
||||
style={{
|
||||
height: `${Math.max(10, heightPercent)}%`,
|
||||
backgroundColor: getProgressColor(sample.usagePercent),
|
||||
}}
|
||||
title={`${new Date(sample.timestamp).toLocaleTimeString()}: ${formatBytes(sample.usage)} (${sample.usagePercent.toFixed(1)}%)`}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* OOM kill counter */}
|
||||
{hasOomKill && (
|
||||
<span className="memory-oom-counter" title={`${memoryStatus.oomKill} OOM kill${memoryStatus.oomKill > 1 ? 's' : ''}`}>
|
||||
💀 {memoryStatus.oomKill}
|
||||
</span>
|
||||
)}
|
||||
|
||||
{/* Swap indicator */}
|
||||
{hasSwap && (
|
||||
<span className="memory-swap-indicator" title={`Swap: ${formatBytes(memoryStatus.cgroupSwapUsage)}`}>
|
||||
🔁
|
||||
</span>
|
||||
)}
|
||||
|
||||
<style>{`
|
||||
.system-memory-indicator {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 4px 12px;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
transition: background 0.2s;
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
.system-memory-indicator:hover {
|
||||
background: var(--bg-hover, rgba(0, 0, 0, 0.1));
|
||||
}
|
||||
|
||||
.system-memory-indicator.system-memory-loading {
|
||||
color: var(--text-secondary, #666);
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.system-memory-indicator.system-memory-error {
|
||||
color: #f44336;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.memory-progress-bar {
|
||||
position: relative;
|
||||
width: 80px;
|
||||
height: 8px;
|
||||
background: var(--bg-secondary, #e0e0e0);
|
||||
border-radius: 4px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.memory-progress-fill {
|
||||
height: 100%;
|
||||
background: currentColor;
|
||||
transition: width 0.5s ease;
|
||||
}
|
||||
|
||||
.memory-text {
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
color: var(--text-primary, #333);
|
||||
min-width: 50px;
|
||||
}
|
||||
|
||||
.memory-sparkline {
|
||||
display: flex;
|
||||
align-items: flex-end;
|
||||
gap: 1px;
|
||||
height: 16px;
|
||||
width: 60px;
|
||||
}
|
||||
|
||||
.sparkline-bar {
|
||||
flex: 1;
|
||||
min-width: 1px;
|
||||
max-width: 3px;
|
||||
border-radius: 1px 1px 0 0;
|
||||
transition: height 0.5s ease, background-color 0.5s ease;
|
||||
}
|
||||
|
||||
.memory-oom-counter {
|
||||
font-size: 12px;
|
||||
color: #d32f2f;
|
||||
font-weight: 600;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.memory-swap-indicator {
|
||||
font-size: 14px;
|
||||
opacity: 0.8;
|
||||
}
|
||||
|
||||
.memory-error-icon {
|
||||
margin-right: 4px;
|
||||
}
|
||||
|
||||
.memory-error-text {
|
||||
font-size: 12px;
|
||||
}
|
||||
`}</style>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default SystemMemoryIndicator;
|
||||
|
|
@ -361,11 +361,12 @@ body {
|
|||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
padding: 0.5rem 1rem;
|
||||
background: var(--bg-tertiary);
|
||||
border-bottom: 1px solid var(--border-color);
|
||||
padding: 0;
|
||||
background: transparent;
|
||||
border: none;
|
||||
font-size: 0.75rem;
|
||||
flex-wrap: wrap;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.fleet-summary-item {
|
||||
|
|
@ -404,6 +405,24 @@ body {
|
|||
opacity: 0.5;
|
||||
}
|
||||
|
||||
/* Fleet Header - combines FleetSummaryBar and SystemMemoryIndicator */
|
||||
.fleet-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
padding: 0.5rem 1rem;
|
||||
background: var(--bg-tertiary);
|
||||
border-bottom: 1px solid var(--border-color);
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.fleet-header-separator {
|
||||
width: 1px;
|
||||
height: 20px;
|
||||
background: var(--border-color);
|
||||
opacity: 0.5;
|
||||
}
|
||||
|
||||
.main-content {
|
||||
display: grid;
|
||||
grid-template-columns: 300px 1fr;
|
||||
|
|
|
|||
|
|
@ -1740,6 +1740,14 @@ export function createWebServer(options: WebServerOptions): WebServer {
|
|||
}
|
||||
|
||||
emitter.emit('start');
|
||||
|
||||
// Start the background memory sampler
|
||||
import('../systemCgroupMonitor.js').then(({ startMemorySampler }) => {
|
||||
startMemorySampler(10000); // Sample every 10 seconds
|
||||
console.log('Memory sampler started (10s interval)');
|
||||
}).catch(err => {
|
||||
console.error('Failed to start memory sampler:', err);
|
||||
});
|
||||
});
|
||||
|
||||
// Second HTTP listener for OTLP/HTTP traffic (port 4318 by convention)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue