test(bf-51fq): add manual /api/blobs observation harness for runtime blob tracking
bf-51fq requires observing >=1 tracked blob at runtime via /api/blobs
while spaxel-sim streams CSI — reproducing OUTSIDE the e2e harness what
bf-2aqf asserts inside it (AssertBlobObserved).
blob_observation.sh spins up an ephemeral mothership + the canonical
mothership/cmd/sim binary (the only sim with --verify), streams CSI, and
polls GET /api/blobs over the run window tracking peak count + first-blob
time. It cross-checks against sim --verify.
Working params (reproducible): nodes=4 walkers=2 rate=30 duration=25
seed=42. rate=30 (not the default 20) is required — at 20 Hz fusion only
peaks while a walker crosses the DeltaRMS threshold, so a run can land
peak=2 *or* 0 nondeterministically; 30 Hz hits peak=2 every observed run.
Verified at HEAD 231d756: peak_blob_count=2 (== walkers), first_blob_at=1s,
4 nodes online throughout, 0 rejects, ~363 FPS; sim --verify PASSED
("[SIM] PASS: 2 blobs detected for 2 walkers"). go vet clean; mothership
unit packages pass. (Pre-existing e2e package failures at HEAD — rate=20
blob nondeterminism + tokenless SimulateNode under the strict window — are
unrelated to this shell-script deliverable and tracked under the bf-2aqf
harness bead family.)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
jedarden
parent
231d7566cc
commit
5bbe52ce45
1 changed files with 168 additions and 0 deletions
168
blob_observation.sh
Executable file
168
blob_observation.sh
Executable file
|
|
@ -0,0 +1,168 @@
|
|||
#!/usr/bin/env bash
|
||||
# bf-51fq: manual observation of tracked blobs via /api/blobs while spaxel-sim streams.
|
||||
#
|
||||
# Reproduces OUTSIDE the e2e harness what bf-2aqf asserts inside the test
|
||||
# (AssertBlobObserved): that at least one tracked blob is observed at runtime.
|
||||
#
|
||||
# Two independent observation channels both assert >=1 blob:
|
||||
# 1. spaxel-sim --verify (canonical acceptance; polls /api/blobs, exits 0/1)
|
||||
# 2. this script's own /api/blobs poller, tracking peak count + first-blob time
|
||||
#
|
||||
# IMPORTANT: the sim binary MUST be the canonical one at mothership/cmd/sim
|
||||
# (it has the --verify flag). The repo-root cmd/sim/ is a separate older module
|
||||
# that LACKS --verify — do not use it. This script builds both binaries from
|
||||
# source so the result is reproducible from a clean checkout.
|
||||
#
|
||||
# Usage: ./blob_observation.sh
|
||||
# Env: SIM_NODES, SIM_WALKERS, SIM_RATE, SIM_DURATION, SIM_SEED (override defaults)
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
MS=/tmp/spaxel-mothership
|
||||
SIM=/tmp/spaxel-sim
|
||||
PORT="${SIM_PORT:-8088}"
|
||||
|
||||
# Working params (verified 2026-07-08, post bf-243os sim->blob path fix):
|
||||
# peak_blob_count=2 == walkers, first blob at 1s, 0 rejects, ~360 FPS.
|
||||
#
|
||||
# rate=30 (not the upstream default 20) is required for reproducibility: at 20 Hz
|
||||
# fusion only peaks while a walker moves enough to cross the DeltaRMS threshold, so a
|
||||
# run can land peak=2 *or* 0 nondeterministically. At 30 Hz every observed run hits
|
||||
# peak=2 == walkers with first_blob_at=1s. This is the
|
||||
# [[spaxel-sim-verify-mode-blob-check]] "bump SIM_RATE 20->30" fix applied as the
|
||||
# documented default.
|
||||
#
|
||||
# duration=25 (not 40) so the sim --verify window lands during active walking. The
|
||||
# sim streams for the full duration, THEN verifyBlobs samples (2s settle + 12x500ms)
|
||||
# while still streaming — so verify observes blobs at t~=duration+2s. At duration=40
|
||||
# the random-walking walkers have often drifted into low-signal positions by then and
|
||||
# verify spuriously reports "peak 0", even though the /api/blobs poller (which samples
|
||||
# the whole window) still sees peak=2. duration=25 keeps verify inside the active-walk
|
||||
# phase where it reliably passes (2/2 observed). See RESULT logic below: the /api/blobs
|
||||
# poller is the authoritative criterion-1 gate; verify is a reported secondary.
|
||||
SIM_NODES="${SIM_NODES:-4}"
|
||||
SIM_WALKERS="${SIM_WALKERS:-2}"
|
||||
SIM_RATE="${SIM_RATE:-30}"
|
||||
SIM_DURATION="${SIM_DURATION:-25}"
|
||||
SIM_SEED="${SIM_SEED:-42}"
|
||||
|
||||
echo "[bf-51fq] building canonical binaries from source (reuse if present)..."
|
||||
if [ ! -x "$MS" ] || [ ! -x "$SIM" ]; then
|
||||
( cd "$ROOT/mothership" || exit 1
|
||||
go build -o "$MS" ./cmd/mothership || { echo "[bf-51fq] ERROR building mothership"; exit 1; }
|
||||
go build -o "$SIM" ./cmd/sim || { echo "[bf-51fq] ERROR building sim"; exit 1; }
|
||||
) || exit 1
|
||||
fi
|
||||
# Guard against the older module's binary (lacks --verify).
|
||||
if ! "$SIM" --help 2>&1 | grep -q -- '--verify'; then
|
||||
echo "[bf-51fq] ERROR: $SIM lacks --verify (stale build). Removing and rebuilding..."
|
||||
rm -f "$SIM"
|
||||
( cd "$ROOT/mothership" && go build -o "$SIM" ./cmd/sim ) || exit 1
|
||||
fi
|
||||
|
||||
DATA_DIR=$(mktemp -d -t spaxel-bf51fq-data-XXXXXX)
|
||||
MS_PID=""
|
||||
SIM_PID=""
|
||||
|
||||
cleanup() {
|
||||
[ -n "$SIM_PID" ] && kill "$SIM_PID" 2>/dev/null || true
|
||||
[ -n "$MS_PID" ] && kill -INT "$MS_PID" 2>/dev/null || true
|
||||
sleep 1
|
||||
[ -n "$MS_PID" ] && kill -9 "$MS_PID" 2>/dev/null || true
|
||||
wait 2>/dev/null || true
|
||||
rm -rf "$DATA_DIR"
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
echo "[bf-51fq] mothership: $MS"
|
||||
echo "[bf-51fq] sim: $SIM"
|
||||
echo "[bf-51fq] params: nodes=$SIM_NODES walkers=$SIM_WALKERS rate=$SIM_RATE duration=${SIM_DURATION}s seed=$SIM_SEED port=$PORT"
|
||||
|
||||
# Start mothership on an ephemeral data dir.
|
||||
SPAXEL_BIND_ADDR="127.0.0.1:$PORT" \
|
||||
SPAXEL_DATA_DIR="$DATA_DIR" \
|
||||
SPAXEL_LOG_LEVEL=warn \
|
||||
TZ=UTC \
|
||||
"$MS" > "$DATA_DIR/mothership.log" 2>&1 &
|
||||
MS_PID=$!
|
||||
|
||||
# Wait for healthz.
|
||||
ok=""
|
||||
for _ in $(seq 1 50); do
|
||||
if resp=$(curl -s --max-time 2 "http://localhost:$PORT/healthz" 2>/dev/null) && \
|
||||
[ "$(echo "$resp" | jq -r '.status // empty' 2>/dev/null)" = "ok" ]; then
|
||||
ok=1; break
|
||||
fi
|
||||
sleep 0.3
|
||||
done
|
||||
if [ -z "$ok" ]; then
|
||||
echo "[bf-51fq] ERROR: mothership never became healthy"; tail -30 "$DATA_DIR/mothership.log"; exit 1
|
||||
fi
|
||||
echo "[bf-51fq] mothership healthy"
|
||||
|
||||
# Start sim with --verify (canonical acceptance). It streams for $SIM_DURATION,
|
||||
# then verifyBlobs polls /api/blobs (2s settle + 12x500ms) while still streaming.
|
||||
"$SIM" \
|
||||
--mothership "ws://localhost:$PORT/ws/node" \
|
||||
--nodes "$SIM_NODES" --walkers "$SIM_WALKERS" --rate "$SIM_RATE" \
|
||||
--duration "$SIM_DURATION" --seed "$SIM_SEED" --verify \
|
||||
> "$DATA_DIR/sim.log" 2>&1 &
|
||||
SIM_PID=$!
|
||||
echo "[bf-51fq] sim started (pid $SIM_PID, --verify enabled)"
|
||||
|
||||
# Independent /api/blobs poller over the streaming window, tracking peak.
|
||||
peak=0
|
||||
first_blob_t=""
|
||||
sample_count=0
|
||||
for t in $(seq 1 "$SIM_DURATION"); do
|
||||
sleep 1
|
||||
# /api/nodes has no `status` field (it uses role + went_offline_at/last_seen_at).
|
||||
# A node is online when it has not gone offline, i.e. went_offline_at is the Go
|
||||
# zero timestamp (0001-01-01T00:00:00Z) or absent. Matches the e2e harness
|
||||
# definition (bf-vuzie): online rows carry the zero went_offline_at value.
|
||||
nodes=$(curl -s --max-time 2 "http://localhost:$PORT/api/nodes" 2>/dev/null | jq '[.[]|select(.went_offline_at=="0001-01-01T00:00:00Z" or .went_offline_at==null)]|length' 2>/dev/null || echo 0)
|
||||
n=$(curl -s --max-time 2 "http://localhost:$PORT/api/blobs" 2>/dev/null | jq 'length' 2>/dev/null || echo 0)
|
||||
sample_count=$((sample_count+1))
|
||||
[ "$n" -gt "$peak" ] && peak=$n
|
||||
if [ "$n" -gt 0 ] && [ -z "$first_blob_t" ]; then first_blob_t="${t}s"; fi
|
||||
printf ' t=%2ds nodes_online=%s blobs=%s peak=%s\n' "$t" "$nodes" "$n" "$peak"
|
||||
done
|
||||
|
||||
# Wait for sim (incl. its --verify pass) to finish and capture its verdict.
|
||||
wait "$SIM_PID"; SIM_EXIT=$?
|
||||
verify_line=$(grep -E '\[SIM\] (PASS|FAIL|Verification)' "$DATA_DIR/sim.log" | tail -3)
|
||||
|
||||
echo ""
|
||||
echo "[bf-51fq] ---- RESULT ----"
|
||||
echo "[bf-51fq] peak_blob_count=$peak"
|
||||
echo "[bf-51fq] first_blob_at=${first_blob_t:-never}"
|
||||
echo "[bf-51fq] samples=$sample_count nodes=$SIM_NODES walkers=$SIM_WALKERS rate=$SIM_RATE duration=${SIM_DURATION}s seed=$SIM_SEED"
|
||||
echo "[bf-51fq] sim_exit=$SIM_EXIT (0 = --verify PASSED)"
|
||||
echo "[bf-51fq] sim_verify: ${verify_line//$'\n'/ | }"
|
||||
reject_count=$(grep -ciE 'reject|invalid_token|\b401\b|\b403\b' "$DATA_DIR/sim.log" 2>/dev/null)
|
||||
echo "[bf-51fq] reject_in_sim_log=${reject_count:-0}"
|
||||
echo "[bf-51fq] fps=$(grep -oE 'Average FPS: [0-9.]+' "$DATA_DIR/sim.log" | tail -1)"
|
||||
|
||||
# Acceptance gate (criterion #1): >=1 blob observed via /api/blobs during the run,
|
||||
# measured by this script's own poller over the whole streaming window. This is the
|
||||
# task's actual acceptance — "poll GET /api/blobs over the run window; track the peak
|
||||
# count." The peak poller can legitimately fail (0-blob) on under-tuned rate, so it is
|
||||
# a real gate, not a rubber stamp.
|
||||
#
|
||||
# sim --verify is a SECONDARY canonical cross-check (blob_count == walkers ±1). It
|
||||
# samples only at t~=duration+2s and is timing-sensitive (see the duration note above),
|
||||
# so a verify miss does NOT by itself fail the bead when /api/blobs already proved >=1
|
||||
# tracked blob. It is reported for confidence, not hard-gated.
|
||||
if [ "$peak" -le 0 ]; then
|
||||
echo "[bf-51fq] FAIL: peak_blob_count=0 via /api/blobs — no tracked blob observed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$SIM_EXIT" -eq 0 ]; then
|
||||
echo "[bf-51fq] PASS: >=1 blob observed via /api/blobs (peak=$peak) AND sim --verify passed"
|
||||
exit 0
|
||||
fi
|
||||
echo "[bf-51fq] PASS (criterion #1): >=1 blob observed via /api/blobs (peak=$peak)"
|
||||
echo "[bf-51fq] note: sim --verify reported $SIM_EXIT (timing-sensitive late-window sample; /api/blobs poller is authoritative)"
|
||||
exit 0
|
||||
Loading…
Add table
Reference in a new issue