fix(bf-27e4): unify stuck detection metric with beadsCompleted
Fix beadsCompleted vs stuck detection metric discrepancy in /api/workers response. Problem: - /api/workers returned contradictory data: beadsCompleted=285 (counts bead.released events) but stuck=true with "only 1 completion(s)" reason - stuck detection counted a different metric while beadsCompleted counted bead.released - When all beads timed out and were deferred, beadsCompleted incremented but stuck detector saw zero success outcomes and flagged the worker as stuck Solution: - Separated beadsCompleted (bead.completed events only) from beadsReleased (bead.released with release_success, includes timed-out/deferred) - Updated stuck detection to use beadsCompleted for successful completions - Added beadsReleased counter to track all processed beads (including timeouts) - Improved stuck reason to distinguish "processed" vs "successful completions" - Updated evidence to show both metrics for clarity Now a worker that processes 100 beads (all timed out) will show: - beadsReleased: 100 - beadsCompleted: 0 - stuckReason: "Running for Xm with 100 processed but 0 successful completions (all timed out/deferred)" Acceptance criteria met: - A worker processing 100 timed-out beads shows clearly that it processed 100 but completed 0 successfully - The stuck flag fires with accurate reason text Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
c047131e09
commit
0473904434
4 changed files with 33 additions and 10 deletions
|
|
@ -622,6 +622,7 @@ export class InMemoryEventStore implements EventStore {
|
|||
id: event.worker,
|
||||
status: 'active',
|
||||
beadsCompleted: 0,
|
||||
beadsReleased: 0,
|
||||
firstSeen: event.ts,
|
||||
lastActivity: event.ts,
|
||||
activeFiles: [],
|
||||
|
|
@ -695,6 +696,7 @@ export class InMemoryEventStore implements EventStore {
|
|||
needleEvent === 'worker.draining'
|
||||
) {
|
||||
worker.status = 'idle';
|
||||
// bead.completed = true successful completion, not just release
|
||||
if (needleEvent === 'bead.completed' && event.bead) {
|
||||
worker.beadsCompleted++;
|
||||
}
|
||||
|
|
@ -708,8 +710,9 @@ export class InMemoryEventStore implements EventStore {
|
|||
worker.currentBead = null;
|
||||
if (event['reason'] === 'release_success') {
|
||||
worker.status = 'idle';
|
||||
// bead.released with release_success includes timed-out/deferred beads
|
||||
if (event.bead) {
|
||||
worker.beadsCompleted++;
|
||||
worker.beadsReleased++;
|
||||
}
|
||||
worker.activeFiles = [];
|
||||
worker.activeDirectories = [];
|
||||
|
|
|
|||
|
|
@ -284,20 +284,35 @@ function detectLongRunning(
|
|||
if (runningTime > opts.longRunningThresholdMs) {
|
||||
const minutes = Math.floor(runningTime / 60000);
|
||||
|
||||
// Use worker.beadsCompleted (counts bead.completed and bead.released with release_success)
|
||||
// instead of text-based message matching
|
||||
// Use worker.beadsCompleted (counts only bead.completed events)
|
||||
// worker.beadsReleased counts bead.released with release_success (includes timed-out/deferred)
|
||||
const completions = worker.beadsCompleted;
|
||||
const released = worker.beadsReleased;
|
||||
|
||||
if (completions < 2) {
|
||||
// Build evidence string showing both completed and released counts
|
||||
const evidence = [
|
||||
`Beads successfully completed: ${worker.beadsCompleted}`,
|
||||
`Beads released (including timed-out): ${released || 0}`,
|
||||
`Total events in window: ${events.length}`,
|
||||
];
|
||||
|
||||
// If we have released beads but no completions, note that in the reason
|
||||
let reason: string;
|
||||
if (released > 0 && completions === 0) {
|
||||
reason = `Running for ${minutes}m with ${released} processed but 0 successful completions (all timed out/deferred)`;
|
||||
} else {
|
||||
reason = `Running for ${minutes}m with only ${completions} successful completion(s)`;
|
||||
}
|
||||
|
||||
return {
|
||||
type: 'long_running',
|
||||
reason: `Running for ${minutes}m with only ${completions} successful completion(s)`,
|
||||
reason,
|
||||
severity: minutes >= 20 ? 'critical' : 'warning',
|
||||
evidence: [
|
||||
`Beads completed: ${worker.beadsCompleted}`,
|
||||
`Total events in window: ${events.length}`,
|
||||
],
|
||||
suggestion: 'Consider breaking task into smaller pieces',
|
||||
evidence,
|
||||
suggestion: released > 0 && completions === 0
|
||||
? 'Beads are timing out before completion — check task complexity or resource constraints'
|
||||
: 'Consider breaking task into smaller pieces',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -554,9 +554,12 @@ export interface WorkerInfo {
|
|||
/** Last event received */
|
||||
lastEvent?: LogEvent;
|
||||
|
||||
/** Total beads completed */
|
||||
/** Total beads completed (bead.completed events only, excludes timed-out/deferred releases) */
|
||||
beadsCompleted: number;
|
||||
|
||||
/** Total beads released with release_success (includes timed-out/deferred beads) */
|
||||
beadsReleased: number;
|
||||
|
||||
/** First seen timestamp */
|
||||
firstSeen: number;
|
||||
|
||||
|
|
|
|||
|
|
@ -245,6 +245,7 @@ interface WorkerTrackingData {
|
|||
|
||||
// Bead tracking
|
||||
beadsCompleted: number;
|
||||
beadsReleased: number;
|
||||
beadStartTimes: Map<string, number>; // beadId -> start timestamp
|
||||
beadCompletionTimes: number[]; // completion durations in ms
|
||||
|
||||
|
|
@ -673,6 +674,7 @@ export class WorkerAnalytics implements WorkerAnalyticsStore {
|
|||
lastSeen: timestamp,
|
||||
lastActivity: timestamp,
|
||||
beadsCompleted: 0,
|
||||
beadsReleased: 0,
|
||||
beadStartTimes: new Map(),
|
||||
beadCompletionTimes: [],
|
||||
errorCount: 0,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue