jedarden/spaxel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add end-to-end integration test harness

Browse source 
Implements a comprehensive e2e test system that:
- Starts mothership container/binary
- Waits for /healthz with 15s timeout
- Handles PIN auth setup if needed
- Runs CSI simulator against mothership
- Asserts during run (health, nodes online, blob detection)
- Validates frame rate doesn't drop >20%
- Asserts detection events recorded

Components added:
- mothership/cmd/sim: CSI simulator that generates synthetic frames
- mothership/tests/e2e: Go test suite with WebSocket assertions
- tests/e2e/run.sh: Shell script with comprehensive assertions
- .github/workflows/e2e.yml: CI workflow for automated testing

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jedarden 2026-04-07 12:35:00 -04:00
parent 9810da2ee6
commit 60a21bacb6
4 changed files with 1756 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
.github/workflows/e2e.yml vendored Normal file
View file

@ -0,0 +1,75 @@
name: E2E Tests
on:
push:
branches: [main]
paths:
- 'mothership/**'
- 'tests/**'
- '.github/workflows/e2e.yml'
pull_request:
branches: [main]
paths:
- 'mothership/**'
- 'tests/**'
- '.github/workflows/e2e.yml'
workflow_dispatch:
jobs:
e2e:
name: End-to-End Integration Tests
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.25'
cache-dependency-path: mothership/go.sum
- name: Build mothership
working-directory: mothership
run: go build -v ./cmd/mothership
- name: Build simulator
working-directory: mothership
run: go build -v -o /tmp/spaxel-sim ./cmd/sim
- name: Run e2e tests
working-directory: mothership
run: go test -v -timeout 90s ./tests/e2e/...
docker-e2e:
name: Docker E2E Tests
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.25'
cache-dependency-path: mothership/go.sum
- name: Build Docker image
run: |
docker build -t spaxel-e2e:test .
- name: Build simulator
working-directory: mothership
run: go build -v -o /tmp/spaxel-sim ./cmd/sim
- name: Run shell-based e2e test with Docker
env:
MOTHERSHIP_IMAGE: spaxel-e2e:test
run: |
chmod +x tests/e2e/run.sh
cd mothership
../tests/e2e/run.sh

551
mothership/cmd/sim/main.go Normal file
View file

@ -0,0 +1,551 @@
// Package main provides a CSI simulator for testing the mothership.
// It simulates ESP32 nodes that send synthetic CSI frames via WebSocket.
package main
import (
"encoding/binary"
"encoding/json"
"flag"
"fmt"
"log"
"math"
"math/rand"
"net/http"
"net/url"
"os"
"sync"
"time"
"github.com/gorilla/websocket"
)
const (
// CSI frame constants from the plan
HeaderSize = 24
MaxSubcarriers = 64
DefaultSubcarriers = 52 // Typical HT20
// WiFi wavelength for Fresnel calculations
Wavelength = 0.123 // meters (2.4 GHz)
)
var (
mothershipURL = flag.String("mothership", "ws://localhost:8080/ws/node", "Mothership WebSocket URL")
nodes = flag.Int("nodes", 4, "Number of virtual nodes to simulate")
walkers = flag.Int("walkers", 1, "Number of walking persons to simulate")
rate = flag.Int("rate", 20, "CSI packet rate in Hz")
duration = flag.Duration("duration", 30*time.Second, "Simulation duration")
enableBLE = flag.Bool("ble", false, "Also send simulated BLE advertisements")
seed = flag.Int64("seed", 42, "Random seed for reproducible runs")
spaceWidth = flag.Float64("width", 6.0, "Space width in meters")
spaceDepth = flag.Float64("depth", 5.0, "Space depth in meters")
spaceHeight = flag.Float64("height", 2.5, "Space height in meters")
showFrameRate = flag.Bool("show-frame-rate", true, "Show per-second frame counts to stdout")
verbose = flag.Bool("verbose", false, "Enable verbose logging")
)
// CSIFrame represents a CSI binary frame
type CSIFrame struct {
NodeMAC [6]byte
PeerMAC [6]byte
TimestampUS uint64
RSSI int8
NoiseFloor int8
Channel uint8
NSub uint8
Payload []int8 // Interleaved I,Q pairs
}
// HelloMessage is sent on connection
type HelloMessage struct {
Type string `json:"type"`
MAC string `json:"mac"`
NodeID string `json:"node_id,omitempty"`
FirmwareVersion string `json:"firmware_version"`
Capabilities []string `json:"capabilities"`
Chip string `json:"chip,omitempty"`
FlashMB int `json:"flash_mb,omitempty"`
UptimeMS int64 `json:"uptime_ms,omitempty"`
APBSSID string `json:"ap_bssid,omitempty"`
APChannel int `json:"ap_channel,omitempty"`
}
// HealthMessage is sent every 10 seconds
type HealthMessage struct {
Type string `json:"type"`
MAC string `json:"mac"`
TimestampMS int64 `json:"timestamp_ms"`
FreeHeapBytes int64 `json:"free_heap_bytes"`
WifiRSSIdBm int `json:"wifi_rssi_dbm"`
UptimeMS int64 `json:"uptime_ms"`
TemperatureC float64 `json:"temperature_c,omitempty"`
CSIRateHz int `json:"csi_rate_hz"`
WifiChannel int `json:"wifi_channel"`
IP string `json:"ip,omitempty"`
}
// BLEMessage is sent every 5 seconds
type BLEMessage struct {
Type string `json:"type"`
MAC string `json:"mac"`
TimestampMS int64 `json:"timestamp_ms"`
Devices []BLEDevice `json:"devices"`
}
// BLEDevice represents a simulated BLE device
type BLEDevice struct {
Addr string `json:"addr"`
AddrType string `json:"addr_type,omitempty"`
RSSIdBm int `json:"rssi_dbm"`
Name string `json:"name,omitempty"`
MfrID int `json:"mfr_id,omitempty"`
MfrDataHex string `json:"mfr_data_hex,omitempty"`
}
// VirtualNode represents a simulated ESP32 node
type VirtualNode struct {
mac string
position [3]float64 // x, y, z in meters
conn *websocket.Conn
mu sync.Mutex
connected bool
frameCount int
lastSecond time.Time
secondCount int
}
// Walker represents a simulated person moving through space
type Walker struct {
position [3]float64 // x, y, z in meters
velocity [3]float64 // vx, vy, vz in m/s
mac string // BLE address for this walker
}
func main() {
flag.Parse()
if *seed != 0 {
rand.Seed(*seed)
}
log.Printf("[INFO] CSI Simulator starting")
log.Printf("[INFO] Configuration: nodes=%d, walkers=%d, rate=%d Hz, duration=%s", *nodes, *walkers, *rate, *duration)
log.Printf("[INFO] Space: %.1fx%.1fx%.1f m", *spaceWidth, *spaceDepth, *spaceHeight)
log.Printf("[INFO] Connecting to: %s", *mothershipURL)
// Create virtual nodes at corners and edges of the room
virtualNodes := createVirtualNodes(*nodes, *spaceWidth, *spaceDepth, *spaceHeight)
// Create walkers
walkers := createWalkers(*walkers, *spaceWidth, *spaceDepth, *spaceHeight)
// Start all nodes
var wg sync.WaitGroup
for i := range virtualNodes {
wg.Add(1)
go func(n *VirtualNode) {
defer wg.Done()
if err := n.run(walkers, *rate, *duration, *enableBLE, *verbose); err != nil {
log.Printf("[ERROR] Node %s failed: %v", n.mac, err)
os.Exit(1)
}
}(&virtualNodes[i])
}
// Wait for all nodes to complete or error
wg.Wait()
log.Printf("[INFO] Simulation completed successfully")
if *showFrameRate {
for _, n := range virtualNodes {
log.Printf("[STATS] Node %s: sent %d frames", n.mac, n.frameCount)
}
}
}
// createVirtualNodes positions virtual nodes in the space
func createVirtualNodes(count int, width, depth, height float64) []VirtualNode {
nodes := make([]VirtualNode, count)
for i := 0; i < count; i++ {
// Position nodes around the perimeter and corners
switch i {
case 0:
nodes[i].position = [3]float64{0, 0, height * 0.8} // Top-left, high
case 1:
nodes[i].position = [3]float64{width, 0, height * 0.8} // Top-right, high
case 2:
nodes[i].position = [3]float64{0, depth, height * 0.8} // Bottom-left, high
case 3:
nodes[i].position = [3]float64{width, depth, height * 0.8} // Bottom-right, high
case 4:
nodes[i].position = [3]float64{width / 2, 0, height * 0.3} // Top-middle, low
case 5:
nodes[i].position = [3]float64{width / 2, depth, height * 0.3} // Bottom-middle, low
default:
// Distribute remaining nodes evenly
nodes[i].position = [3]float64{
(float64(i) * width) / float64(count),
(float64(i) * depth) / float64(count),
height * 0.5,
}
}
// Generate MAC address
nodes[i].mac = fmt.Sprintf("AA:BB:CC:DD:%02X:00", i)
}
return nodes
}
// createWalkers creates simulated walkers
func createWalkers(count int, width, depth, height float64) []Walker {
walkers := make([]Walker, count)
for i := range walkers {
// Start in center of room
walkers[i].position = [3]float64{width / 2, depth / 2, 1.7} // 1.7m = average person height
// Random initial velocity
walkers[i].velocity = [3]float64{
(rand.Float64() - 0.5) * 0.5, // -0.25 to +0.25 m/s X
(rand.Float64() - 0.5) * 0.5, // -0.25 to +0.25 m/s Y
0, // Z stays constant
}
// Generate BLE address
walkers[i].mac = fmt.Sprintf("11:22:33:44:55:%02X", i)
}
return walkers
}
// run starts the virtual node simulation
func (n *VirtualNode) run(walkers []Walker, rateHz int, duration time.Duration, enableBLE, verbose bool) error {
// Parse mothership URL
u, err := url.Parse(*mothershipURL)
if err != nil {
return fmt.Errorf("invalid mothership URL: %w", err)
}
// Connect to mothership
dialer := websocket.Dialer{
HandshakeTimeout: 5 * time.Second,
}
conn, _, err := dialer.Dial(u.String(), nil)
if err != nil {
return fmt.Errorf("WebSocket dial failed: %w", err)
}
defer conn.Close()
n.mu.Lock()
n.conn = conn
n.connected = true
n.lastSecond = time.Now()
n.mu.Unlock()
log.Printf("[INFO] Node %s connected to mothership", n.mac)
// Send hello message
uptime := int64(1000) // 1 second
hello := HelloMessage{
Type: "hello",
MAC: n.mac,
NodeID: fmt.Sprintf("sim-node-%s", n.mac),
FirmwareVersion: "0.1.0-sim",
Capabilities: []string{"csi", "tx", "rx"},
Chip: "ESP32-S3",
FlashMB: 16,
UptimeMS: uptime,
}
helloJSON, err := json.Marshal(hello)
if err != nil {
return fmt.Errorf("failed to marshal hello: %w", err)
}
if err := conn.WriteMessage(websocket.TextMessage, helloJSON); err != nil {
return fmt.Errorf("failed to send hello: %w", err)
}
if verbose {
log.Printf("[DEBUG] Node %s sent hello", n.mac)
}
// Wait for role assignment
time.Sleep(100 * time.Millisecond)
// Start ticker for CSI frames
ticker := time.NewTicker(time.Second / time.Duration(rateHz))
defer ticker.Stop()
// Health ticker (every 10 seconds)
healthTicker := time.NewTicker(10 * time.Second)
defer healthTicker.Stop()
// BLE ticker (every 5 seconds)
var bleTicker *time.Ticker
if enableBLE {
bleTicker = time.NewTicker(5 * time.Second)
defer bleTicker.Stop()
}
// Frame rate tracking ticker
var frameRateTicker *time.Ticker
if *showFrameRate {
frameRateTicker = time.NewTicker(time.Second)
defer frameRateTicker.Stop()
}
startTime := time.Now()
frameIndex := uint64(0)
// Main loop
for time.Since(startTime) < duration {
select {
case <-ticker.C:
// Update walker positions
for i := range walkers {
updateWalkerPosition(&walkers[i], *spaceWidth, *spaceDepth)
}
// Generate and send CSI frames for each link
for _, walker := range walkers {
frame := n.generateCSIFrame(walker, frameIndex)
if err := conn.WriteMessage(websocket.BinaryMessage, frame); err != nil {
return fmt.Errorf("failed to send CSI frame: %w", err)
}
n.frameCount++
n.secondCount++
frameIndex++
}
case <-healthTicker.C:
// Send health message
uptime = time.Since(startTime).Milliseconds()
health := HealthMessage{
Type: "health",
MAC: n.mac,
TimestampMS: time.Now().UnixMilli(),
FreeHeapBytes: 204800,
WifiRSSIdBm: -50 - rand.Intn(20), // -50 to -70
UptimeMS: uptime,
TemperatureC: 40 + rand.Float64()*5,
CSIRateHz: rateHz,
WifiChannel: 6,
IP: "192.168.1.100",
}
healthJSON, err := json.Marshal(health)
if err != nil {
log.Printf("[WARN] Failed to marshal health: %v", err)
continue
}
if err := conn.WriteMessage(websocket.TextMessage, healthJSON); err != nil {
return fmt.Errorf("failed to send health: %w", err)
}
if verbose {
log.Printf("[DEBUG] Node %s sent health", n.mac)
}
case <-bleTicker.C:
// Send BLE scan results
if len(walkers) > 0 {
walker := walkers[0] // Use first walker's BLE
ble := BLEMessage{
Type: "ble",
MAC: n.mac,
TimestampMS: time.Now().UnixMilli(),
Devices: []BLEDevice{
{
Addr: walker.mac,
AddrType: "public",
RSSIdBm: -60 - rand.Intn(20),
Name: "SimPhone",
MfrID: 76, // Apple
},
},
}
bleJSON, err := json.Marshal(ble)
if err != nil {
log.Printf("[WARN] Failed to marshal BLE: %v", err)
continue
}
if err := conn.WriteMessage(websocket.TextMessage, bleJSON); err != nil {
return fmt.Errorf("failed to send BLE: %w", err)
}
if verbose {
log.Printf("[DEBUG] Node %s sent BLE scan", n.mac)
}
}
case <-frameRateTicker.C:
// Report frame rate
log.Printf("[STATS] Node %s: %d frames/s", n.mac, n.secondCount)
n.secondCount = 0
}
// Check for reject message
conn.SetReadDeadline(time.Now().Add(100 * time.Millisecond))
_, msg, err := conn.ReadMessage()
if err != nil {
if !websocket.IsTimeout(err) && err.Error() != "EOF" {
return fmt.Errorf("read error: %w", err)
}
} else if len(msg) > 0 && msg[0] == '{' {
// JSON message
var base struct {
Type string `json:"type"`
}
if err := json.Unmarshal(msg, &base); err == nil && base.Type == "reject" {
return fmt.Errorf("node rejected by mothership")
}
}
}
return nil
}
// generateCSIFrame creates a synthetic CSI frame based on walker position
func (n *VirtualNode) generateCSIFrame(walker Walker, frameIndex uint64) []byte {
nSub := DefaultSubcarriers
// Calculate distance to walker
dx := walker.position[0] - n.position[0]
dy := walker.position[1] - n.position[1]
dz := walker.position[2] - n.position[2]
distance := math.Sqrt(dx*dx + dy*dy + dz*dz)
// Calculate path loss for RSSI
// Free space path loss: PL(d) = PL_0 + 10*n*log10(d/d_0)
// PL_0 = 40 dB at d_0 = 1m, n = 2.0
pathLoss := 40 + 20*math.Log10(distance/1.0)
rssi := int8(-30 - pathLoss) // -30 dBm reference
// Add Fresnel zone modulation
// When walker is in a Fresnel zone, amplitude increases
fresnelMod := fresnelModulation(n.position, walker.position)
// Create frame
buf := make([]byte, HeaderSize + nSub*2)
// Node MAC (6 bytes)
macBytes := macToBytes(n.mac)
copy(buf[0:6], macBytes[:])
// Peer MAC (6 bytes) - use walker's simulated MAC
peerMAC := macToBytes(fmt.Sprintf("11:22:33:44:55:%02X", 0))
copy(buf[6:12], peerMAC[:])
// Timestamp (8 bytes, uint64, little-endian)
timestampUS := uint64(frameIndex * 1_000_000 / uint64(*rate))
binary.LittleEndian.PutUint64(buf[12:20], timestampUS)
// RSSI (1 byte, int8)
buf[20] = byte(rssi)
// Noise floor (1 byte, int8)
buf[21] = byte(-95) // Typical noise floor
// Channel (1 byte, uint8)
buf[22] = 6 // Channel 6
// Number of subcarriers (1 byte, uint8)
buf[23] = byte(nSub)
// Generate CSI payload (I, Q pairs)
for k := 0; k < nSub; k++ {
// Base amplitude with Fresnel modulation
amplitude := 30.0 + float64(k)*0.1 + fresnelMod*8.0
// Add subcarrier-dependent phase
phase := float64(k) * 0.2
// Add noise
noise := rand.NormFloat64() * 2.0
// Convert to I, Q
iVal := int8(amplitude*math.Cos(phase) + noise)
qVal := int8(amplitude*math.Sin(phase) + noise)
offset := HeaderSize + k*2
buf[offset] = byte(iVal)
buf[offset+1] = byte(qVal)
}
return buf
}
// fresnelModulation calculates the Fresnel zone modulation factor
func fresnelModulation(nodePos, walkerPos [3]float64) float64 {
// Calculate path length excess
nodeToWalker := math.Sqrt(
math.Pow(walkerPos[0]-nodePos[0], 2) +
math.Pow(walkerPos[1]-nodePos[1], 2) +
math.Pow(walkerPos[2]-nodePos[2], 2))
walkerToPeer := nodeToWalker // Simplified: peer is at same distance
directPath := 5.0 // Simplified direct path
deltaL := nodeToWalker + walkerToPeer - directPath
// Fresnel zone number (λ/2 = 0.0615m)
zone := math.Ceil(deltaL / 0.0615)
// Modulation factor based on zone
// Zone 1: maximum modulation, Zone 5+: minimum
if zone <= 1 {
return 1.0
}
if zone >= 5 {
return 0.0
}
return 1.0 / math.Pow(zone, 2.0)
}
// updateWalkerPosition updates walker position with random walk
func updateWalkerPosition(w *Walker, width, depth float64) {
const dt = 0.05 // 50ms step
// Update position
w.position[0] += w.velocity[0] * dt
w.position[1] += w.velocity[1] * dt
// Bounce off walls
if w.position[0] < 0 || w.position[0] > width {
w.velocity[0] *= -1
w.position[0] = math.Max(0, math.Min(width, w.position[0]))
}
if w.position[1] < 0 || w.position[1] > depth {
w.velocity[1] *= -1
w.position[1] = math.Max(0, math.Min(depth, w.position[1]))
}
// Random velocity perturbation (simulates human motion)
w.velocity[0] += (rand.Float64() - 0.5) * 0.1
w.velocity[1] += (rand.Float64() - 0.5) * 0.1
// Clamp velocity
maxSpeed := 0.5
speed := math.Sqrt(w.velocity[0]*w.velocity[0] + w.velocity[1]*w.velocity[1])
if speed > maxSpeed {
scale := maxSpeed / speed
w.velocity[0] *= scale
w.velocity[1] *= scale
}
}
// macToBytes converts MAC string to bytes
func macToBytes(mac string) [6]byte {
var b [6]byte
fmt.Sscanf(mac, "%02X:%02X:%02X:%02X:%02X:%02X",
&b[0], &b[1], &b[2], &b[3], &b[4], &b[5])
return b
}

781
mothership/tests/e2e/e2e_test.go Normal file
View file

@ -0,0 +1,781 @@
// Package e2e provides end-to-end integration tests for Spaxel.
// These tests start the mothership, run the CSI simulator, and assert on behavior.
package e2e
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"sync"
"testing"
"time"
"github.com/gorilla/websocket"
)
const (
// Default test configuration
DefaultMothershipURL = "ws://localhost:8080/ws/node"
DefaultAPIURL = "http://localhost:8080"
HealthTimeout = 15 * time.Second
SimDuration = 30 * time.Second
TestTimeout = 90 * time.Second
)
// TestHarness manages the e2e test lifecycle
type TestHarness struct {
MothershipCmd *exec.Cmd
SimulatorCmd *exec.Cmd
MothershipURL string
APIURL string
t *testing.T
}
// NewTestHarness creates a new test harness
func NewTestHarness(t *testing.T) *TestHarness {
return &TestHarness{
MothershipURL: DefaultMothershipURL,
APIURL: DefaultAPIURL,
t: t,
}
}
// Start starts the mothership process
func (h *TestHarness) Start(ctx context.Context) error {
// Build mothership first
buildCmd := exec.CommandContext(ctx, "go", "build", "-o", "/tmp/spaxel-mothership-test", "./cmd/mothership")
if output, err := buildCmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to build mothership: %w: %s", err, string(output))
}
// Create temporary data directory
tmpDir, err := os.MkdirTemp("", "spaxel-e2e-*")
if err != nil {
return fmt.Errorf("failed to create temp dir: %w", err)
}
// Start mothership
h.MothershipCmd = exec.CommandContext(ctx, "/tmp/spaxel-mothership-test")
h.MothershipCmd.Env = append(os.Environ(),
"SPAXEL_BIND_ADDR=127.0.0.1:8080",
"SPAXEL_DATA_DIR="+tmpDir,
"SPAXEL_LOG_LEVEL=info",
"TZ=UTC",
)
h.MothershipCmd.Stdout = io.Discard
h.MothershipCmd.Stderr = io.Discard
if err := h.MothershipCmd.Start(); err != nil {
return fmt.Errorf("failed to start mothership: %w", err)
}
h.t.Logf("Mothership started (PID: %d)", h.MothershipCmd.Process.Pid)
// Wait for health check
if err := h.WaitForHealth(ctx); err != nil {
h.Stop()
return fmt.Errorf("health check failed: %w", err)
}
return nil
}
// Stop stops all processes
func (h *TestHarness) Stop() {
if h.MothershipCmd != nil && h.MothershipCmd.Process != nil {
h.MothershipCmd.Process.Signal(os.Interrupt)
h.MothershipCmd.Wait()
}
if h.SimulatorCmd != nil && h.SimulatorCmd.Process != nil {
h.SimulatorCmd.Process.Kill()
h.SimulatorCmd.Wait()
}
}
// WaitForHealth waits for the /healthz endpoint to return ok
func (h *TestHarness) WaitForHealth(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, HealthTimeout)
defer cancel()
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
resp, err := http.Get(h.APIURL + "/healthz")
if err != nil {
continue
}
defer resp.Body.Close()
var health HealthResponse
if err := json.NewDecoder(resp.Body).Decode(&health); err != nil {
continue
}
if health.Status == "ok" {
h.t.Logf("Mothership healthy (uptime: %ds)", health.UptimeS)
return nil
}
}
}
}
// HealthResponse represents the /healthz response
type HealthResponse struct {
Status string `json:"status"`
UptimeS int64 `json:"uptime_s"`
Version string `json:"version"`
NodesOnline int `json:"nodes_online"`
DB string `json:"db"`
LoadLevel int `json:"load_level"`
}
// RunSimulator starts the simulator
func (h *TestHarness) RunSimulator(ctx context.Context, nodes, walkers, rate int, duration time.Duration) error {
// Build simulator
buildCmd := exec.CommandContext(ctx, "go", "build", "-o", "/tmp/spaxel-sim-test", "./cmd/sim")
if output, err := buildCmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to build simulator: %w: %s", err, string(output))
}
// Start simulator
h.SimulatorCmd = exec.CommandContext(ctx, "/tmp/spaxel-sim-test",
"--mothership", h.MothershipURL,
"--nodes", fmt.Sprintf("%d", nodes),
"--walkers", fmt.Sprintf("%d", walkers),
"--rate", fmt.Sprintf("%d", rate),
"--duration", duration.String(),
"--ble",
"--seed", "42",
)
h.SimulatorCmd.Stdout = io.Discard
h.SimulatorCmd.Stderr = io.Discard
if err := h.SimulatorCmd.Start(); err != nil {
return fmt.Errorf("failed to start simulator: %w", err)
}
h.t.Logf("Simulator started (PID: %d)", h.SimulatorCmd.Process.Pid)
return nil
}
// GetNodes retrieves the list of nodes
func (h *TestHarness) GetNodes(ctx context.Context) ([]Node, error) {
resp, err := http.Get(h.APIURL + "/api/nodes")
if err != nil {
return nil, err
}
defer resp.Body.Close()
var nodes []Node
if err := json.NewDecoder(resp.Body).Decode(&nodes); err != nil {
return nil, err
}
return nodes, nil
}
// Node represents a node from the API
type Node struct {
MAC string `json:"mac"`
Name string `json:"name"`
Role string `json:"role"`
Position Position `json:"position"`
FirmwareVersion string `json:"firmware_version"`
Status string `json:"status"`
RSSI int `json:"rssi"`
UptimeS int64 `json:"uptime_s"`
LastSeen int64 `json:"last_seen_ms"`
}
// Position represents a node position
type Position struct {
X float64 `json:"pos_x"`
Y float64 `json:"pos_y"`
Z float64 `json:"pos_z"`
}
// GetEvents retrieves events from the API
func (h *TestHarness) GetEvents(ctx context.Context, eventType string, limit int) (*EventsResponse, error) {
url := h.APIURL + "/api/events"
if eventType != "" {
url += "?type=" + eventType
}
if limit > 0 {
if eventType != "" {
url += "&"
} else {
url += "?"
}
url += fmt.Sprintf("limit=%d", limit)
}
resp, err := http.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
var events EventsResponse
if err := json.NewDecoder(resp.Body).Decode(&events); err != nil {
return nil, err
}
return &events, nil
}
// EventsResponse represents the /api/events response
type EventsResponse struct {
Events []Event `json:"events"`
Cursor string `json:"cursor,omitempty"`
Total int `json:"total,omitempty"`
}
// Event represents a single event
type Event struct {
ID int64 `json:"id"`
TimestampMS int64 `json:"timestamp_ms"`
Type string `json:"type"`
Zone string `json:"zone,omitempty"`
Person string `json:"person,omitempty"`
BlobID int `json:"blob_id,omitempty"`
Detail json.RawMessage `json:"detail_json,omitempty"`
Severity string `json:"severity"`
}
// WatchDashboardWS connects to the dashboard WebSocket and returns blob counts
func (h *TestHarness) WatchDashboardWS(ctx context.Context, duration time.Duration) ([]int, error) {
wsURL := "ws://localhost:8080/ws/dashboard"
conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
if err != nil {
return nil, fmt.Errorf("failed to connect to dashboard WS: %w", err)
}
defer conn.Close()
blobCounts := make([]int, 0)
ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop()
startTime := time.Now()
for time.Since(startTime) < duration {
select {
case <-ctx.Done():
return blobCounts, ctx.Err()
case <-ticker.C:
// Read message with timeout
conn.SetReadDeadline(time.Now().Add(2 * time.Second))
_, message, err := conn.ReadMessage()
if err != nil {
if websocket.IsTimeout(err) || err.Error() == "EOF" {
continue
}
return blobCounts, fmt.Errorf("read error: %w", err)
}
// Parse message
var data map[string]interface{}
if err := json.Unmarshal(message, &data); err != nil {
continue
}
// Check for blobs in snapshot or delta messages
blobCount := 0
if blobs, ok := data["blobs"].([]interface{}); ok {
blobCount = len(blobs)
}
blobCounts = append(blobCounts, blobCount)
}
}
return blobCounts, nil
}
// AssertDuringRun polls assertions during the simulation run
func (h *TestHarness) AssertDuringRun(ctx context.Context, duration time.Duration, expectedNodes int) error {
ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop()
startTime := time.Now()
blobDetected := false
nodesSeenOnline := false
for time.Since(startTime) < duration {
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
elapsed := int(time.Since(startTime).Seconds())
// Check health - assert status=='ok' throughout entire run
resp, err := http.Get(h.APIURL + "/healthz")
if err == nil {
defer resp.Body.Close()
var health HealthResponse
if err := json.NewDecoder(resp.Body).Decode(&health); err == nil {
if health.Status != "ok" {
return fmt.Errorf("health check failed at %ds: status=%s", elapsed, health.Status)
}
}
}
// Check nodes - assert nodes_online == expectedNodes within first 5s
if elapsed <= 5 && !nodesSeenOnline {
nodes, err := h.GetNodes(ctx)
if err == nil {
onlineCount := 0
for _, node := range nodes {
if node.Status == "online" {
onlineCount++
}
}
if onlineCount >= expectedNodes {
h.t.Logf("✓ All %d nodes online within first 5s (elapsed: %ds)", expectedNodes, elapsed)
nodesSeenOnline = true
}
}
}
// Check for blobs - assert blob_count > 0 within first 15s
if elapsed >= 5 && elapsed <= 15 && !blobDetected {
events, err := h.GetEvents(ctx, "detection", 10)
if err == nil && len(events.Events) > 0 {
h.t.Logf("✓ Blob detected within first 15s (found %d detection events at %ds)", len(events.Events), elapsed)
blobDetected = true
}
}
}
}
if !blobDetected {
return fmt.Errorf("no blob detected within first 15s")
}
return nil
}
// SimulateNode simulates a single node connection
func (h *TestHarness) SimulateNode(ctx context.Context, mac string, duration time.Duration) error {
conn, _, err := websocket.DefaultDialer.Dial(h.MothershipURL, nil)
if err != nil {
return fmt.Errorf("failed to connect: %w", err)
}
defer conn.Close()
// Send hello message
hello := map[string]interface{}{
"type": "hello",
"mac": mac,
"node_id": "sim-node-" + mac,
"firmware_version": "0.1.0-sim",
"capabilities": []string{"csi", "tx", "rx"},
"chip": "ESP32-S3",
"flash_mb": 16,
"uptime_ms": 1000,
}
if err := conn.WriteJSON(hello); err != nil {
return fmt.Errorf("failed to send hello: %w", err)
}
// Wait for role assignment
time.Sleep(100 * time.Millisecond)
// Send CSI frames
ticker := time.NewTicker(time.Second / 20) // 20 Hz
defer ticker.Stop()
healthTicker := time.NewTicker(10 * time.Second)
defer healthTicker.Stop()
startTime := time.Now()
frameIndex := uint64(0)
for time.Since(startTime) < duration {
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
// Send CSI frame
frame := generateCSIFrame(mac, frameIndex)
if err := conn.WriteMessage(websocket.BinaryMessage, frame); err != nil {
return err
}
frameIndex++
case <-healthTicker.C:
// Send health message
health := map[string]interface{}{
"type": "health",
"mac": mac,
"timestamp_ms": time.Now().UnixMilli(),
"free_heap_bytes": 204800,
"wifi_rssi_dbm": -60,
"uptime_ms": time.Since(startTime).Milliseconds(),
"temperature_c": 42.0,
"csi_rate_hz": 20,
"wifi_channel": 6,
}
if err := conn.WriteJSON(health); err != nil {
return err
}
}
// Check for reject message
conn.SetReadDeadline(time.Now().Add(100 * time.Millisecond))
_, msg, err := conn.ReadMessage()
if err != nil {
if !websocket.IsTimeout(err) {
return err
}
} else if len(msg) > 0 && msg[0] == '{' {
var base struct {
Type string `json:"type"`
}
if err := json.Unmarshal(msg, &base); err == nil && base.Type == "reject" {
return fmt.Errorf("node rejected")
}
}
}
return nil
}
// generateCSIFrame creates a synthetic CSI frame
func generateCSIFrame(mac string, frameIndex uint64) []byte {
const (
HeaderSize = 24
DefaultNSub = 52
)
buf := make([]byte, HeaderSize+DefaultNSub*2)
// Parse MAC to bytes
var macBytes [6]byte
fmt.Sscanf(mac, "%02X:%02X:%02X:%02X:%02X:%02X",
&macBytes[0], &macBytes[1], &macBytes[2], &macBytes[3], &macBytes[4], &macBytes[5])
// Node MAC
copy(buf[0:6], macBytes[:])
// Peer MAC (use a fake peer)
peerMAC := [6]byte{0x11, 0x22, 0x33, 0x44, 0x55, 0x00}
copy(buf[6:12], peerMAC[:])
// Timestamp
timestampUS := frameIndex * 50000 // 20 Hz = 50ms
buf[12] = byte(timestampUS)
buf[13] = byte(timestampUS >> 8)
buf[14] = byte(timestampUS >> 16)
buf[15] = byte(timestampUS >> 24)
buf[16] = byte(timestampUS >> 32)
buf[17] = byte(timestampUS >> 40)
buf[18] = byte(timestampUS >> 48)
buf[19] = byte(timestampUS >> 56)
// RSSI
buf[20] = 0xdc // -40 dBm
// Noise floor
buf[21] = 0xa1 // -95 dBm
// Channel
buf[22] = 6
// Number of subcarriers
buf[23] = DefaultNSub
// Generate CSI payload (I, Q pairs)
for k := 0; k < DefaultNSub; k++ {
amplitude := 30.0 + float64(k)*0.1
phase := float64(k) * 0.2
iVal := int8(amplitude * 0.707) // cos(45deg) ~= 0.707
qVal := int8(amplitude * 0.707)
offset := HeaderSize + k*2
buf[offset] = byte(iVal)
buf[offset+1] = byte(qVal)
}
return buf
}
// TestMothershipHealth tests that the mothership starts and becomes healthy
func TestMothershipHealth(t *testing.T) {
if testing.Short() {
t.Skip("skipping e2e test in short mode")
}
ctx, cancel := context.WithTimeout(context.Background(), TestTimeout)
defer cancel()
h := NewTestHarness(t)
defer h.Stop()
if err := h.Start(ctx); err != nil {
t.Fatalf("Failed to start mothership: %v", err)
}
// Check health endpoint
resp, err := http.Get(h.APIURL + "/healthz")
if err != nil {
t.Fatalf("Failed to get health: %v", err)
}
defer resp.Body.Close()
var health HealthResponse
if err := json.NewDecoder(resp.Body).Decode(&health); err != nil {
t.Fatalf("Failed to decode health: %v", err)
}
if health.Status != "ok" {
t.Errorf("Expected status ok, got %s", health.Status)
}
}
// TestSimulatorConnection tests that the simulator can connect
func TestSimulatorConnection(t *testing.T) {
if testing.Short() {
t.Skip("skipping e2e test in short mode")
}
ctx, cancel := context.WithTimeout(context.Background(), TestTimeout)
defer cancel()
h := NewTestHarness(t)
defer h.Stop()
if err := h.Start(ctx); err != nil {
t.Fatalf("Failed to start mothership: %v", err)
}
// Run simulator for 10 seconds
if err := h.RunSimulator(ctx, 2, 1, 20, 10*time.Second); err != nil {
t.Fatalf("Failed to run simulator: %v", err)
}
// Wait a bit for nodes to connect
time.Sleep(2 * time.Second)
// Check nodes are online
nodes, err := h.GetNodes(ctx)
if err != nil {
t.Fatalf("Failed to get nodes: %v", err)
}
onlineCount := 0
for _, node := range nodes {
if node.Status == "online" {
onlineCount++
}
}
if onlineCount < 2 {
t.Errorf("Expected at least 2 nodes online, got %d", onlineCount)
}
t.Logf("Found %d/%d nodes online", onlineCount, len(nodes))
}
// TestDetectionEvents tests that detection events are generated
func TestDetectionEvents(t *testing.T) {
if testing.Short() {
t.Skip("skipping e2e test in short mode")
}
ctx, cancel := context.WithTimeout(context.Background(), TestTimeout)
defer cancel()
h := NewTestHarness(t)
defer h.Stop()
if err := h.Start(ctx); err != nil {
t.Fatalf("Failed to start mothership: %v", err)
}
// Run simulator
duration := 15 * time.Second
if err := h.RunSimulator(ctx, 4, 2, 20, duration); err != nil {
t.Fatalf("Failed to run simulator: %v", err)
}
// Wait for simulation to complete
time.Sleep(duration + 2*time.Second)
// Check for detection events
events, err := h.GetEvents(ctx, "detection", 100)
if err != nil {
t.Fatalf("Failed to get events: %v", err)
}
if len(events.Events) == 0 {
t.Error("Expected at least 1 detection event, got 0")
}
t.Logf("Found %d detection events", len(events.Events))
}
// TestConcurrentNodes tests multiple concurrent node connections
func TestConcurrentNodes(t *testing.T) {
if testing.Short() {
t.Skip("skipping e2e test in short mode")
}
ctx, cancel := context.WithTimeout(context.Background(), TestTimeout)
defer cancel()
h := NewTestHarness(t)
defer h.Stop()
if err := h.Start(ctx); err != nil {
t.Fatalf("Failed to start mothership: %v", err)
}
// Simulate 4 concurrent nodes
var wg sync.WaitGroup
nodeMACs := []string{
"AA:BB:CC:DD:00:01",
"AA:BB:CC:DD:00:02",
"AA:BB:CC:DD:00:03",
"AA:BB:CC:DD:00:04",
}
duration := 10 * time.Second
for _, mac := range nodeMACs {
wg.Add(1)
go func(mac string) {
defer wg.Done()
if err := h.SimulateNode(ctx, mac, duration); err != nil {
t.Errorf("Node %s failed: %v", mac, err)
}
}(mac)
}
wg.Wait()
// Check all nodes are online
nodes, err := h.GetNodes(ctx)
if err != nil {
t.Fatalf("Failed to get nodes: %v", err)
}
onlineCount := 0
for _, node := range nodes {
if node.Status == "online" {
onlineCount++
}
}
if onlineCount < 4 {
t.Errorf("Expected at least 4 nodes online, got %d", onlineCount)
}
t.Logf("Successfully connected %d nodes", onlineCount)
}
// TestDashboardWebSocket tests the dashboard WebSocket connection
func TestDashboardWebSocket(t *testing.T) {
if testing.Short() {
t.Skip("skipping e2e test in short mode")
}
ctx, cancel := context.WithTimeout(context.Background(), TestTimeout)
defer cancel()
h := NewTestHarness(t)
defer h.Stop()
if err := h.Start(ctx); err != nil {
t.Fatalf("Failed to start mothership: %v", err)
}
// Run simulator for 10 seconds
if err := h.RunSimulator(ctx, 2, 1, 20, 10*time.Second); err != nil {
t.Fatalf("Failed to run simulator: %v", err)
}
// Watch dashboard WebSocket for blob data
blobCounts, err := h.WatchDashboardWS(ctx, 10*time.Second)
if err != nil {
t.Fatalf("Failed to watch dashboard WS: %v", err)
}
t.Logf("Received %d blob count updates", len(blobCounts))
}
// TestFullE2EIntegration runs a comprehensive end-to-end test
func TestFullE2EIntegration(t *testing.T) {
if testing.Short() {
t.Skip("skipping e2e test in short mode")
}
ctx, cancel := context.WithTimeout(context.Background(), TestTimeout)
defer cancel()
h := NewTestHarness(t)
defer h.Stop()
if err := h.Start(ctx); err != nil {
t.Fatalf("Failed to start mothership: %v", err)
}
// Run simulator with 4 nodes, 2 walkers
simDuration := 30 * time.Second
if err := h.RunSimulator(ctx, 4, 2, 20, simDuration); err != nil {
t.Fatalf("Failed to run simulator: %v", err)
}
// Assert during run
if err := h.AssertDuringRun(ctx, simDuration, 4); err != nil {
t.Fatalf("Assertion failed during run: %v", err)
}
// Wait for simulator to complete
time.Sleep(simDuration + 2*time.Second)
// Assert after run: check detection events
events, err := h.GetEvents(ctx, "detection", 100)
if err != nil {
t.Fatalf("Failed to get events: %v", err)
}
if len(events.Events) < 1 {
t.Errorf("Expected at least 1 detection event, got %d", len(events.Events))
}
t.Logf("✓ Full E2E integration test passed with %d detection events", len(events.Events))
}
// TestMain runs the test suite
func TestMain(m *testing.M) {
// Build binaries before running tests
if os.Getenv("GO_BUILD_SKIP") == "" {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
// Build mothership
if err := exec.CommandContext(ctx, "go", "build", "./cmd/mothership").Run(); err != nil {
fmt.Fprintf(os.Stderr, "Failed to build mothership: %v\n", err)
os.Exit(1)
}
// Build simulator
if err := exec.CommandContext(ctx, "go", "build", "./cmd/sim").Run(); err != nil {
fmt.Fprintf(os.Stderr, "Failed to build simulator: %v\n", err)
os.Exit(1)
}
}
os.Exit(m.Run())
}

349
tests/e2e/run.sh Executable file
View file

@ -0,0 +1,349 @@
#!/bin/bash
# End-to-end integration test harness for Spaxel
# Starts the mothership, runs the CSI simulator, and asserts on behavior
set -euo pipefail
# Configuration
MOTHERSHIP_IMAGE="${MOTHERSHIP_IMAGE:-ronaldraygun/spaxel:latest}"
MOTHERSHIP_CONTAINER="spaxel-e2e-test"
MOTHERSHIP_PORT=8080
HEALTH_TIMEOUT=15
SIM_DURATION=30
SIM_NODES=4
SIM_WALKERS=2
SIM_RATE=20
SIM_SEED=42
TEST_TIMEOUT=90
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
# Logging functions
log_info() {
echo -e "${GREEN}[INFO]${NC} $1"
}
log_warn() {
echo -e "${YELLOW}[WARN]${NC} $1"
}
log_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
# Cleanup function
cleanup() {
local exit_code=$?
log_info "Cleaning up..."
# Stop simulator if running
if [ -n "$SIM_PID" ]; then
kill $SIM_PID 2>/dev/null || true
wait $SIM_PID 2>/dev/null || true
fi
# Stop and remove mothership container
docker stop "$MOTHERSHIP_CONTAINER" 2>/dev/null || true
docker rm "$MOTHERSHIP_CONTAINER" 2>/dev/null || true
if [ $exit_code -eq 0 ]; then
log_info "All tests passed!"
else
log_error "Tests failed with exit code $exit_code"
fi
exit $exit_code
}
trap cleanup EXIT INT TERM
# Helper function for HTTP requests with timeout
http_get() {
local url="$1"
local max_attempts="${2:-1}"
local wait="${3:-0}"
for i in $(seq 1 "$max_attempts"); do
if curl -sSf --max-time 5 "$url" 2>/dev/null; then
return 0
fi
if [ $i -lt $max_attempts ]; then
sleep "$wait"
fi
done
return 1
}
# Helper function to check JSON field
json_field() {
local json="$1"
local field="$2"
echo "$json" | jq -r "$field // empty"
}
# Step 1: Start mothership container
log_info "Step 1: Starting mothership container..."
# Check if container is already running and remove it
docker rm -f "$MOTHERSHIP_CONTAINER" 2>/dev/null || true
# Run mothership container
docker run -d \
--name "$MOTHERSHIP_CONTAINER" \
-p "$MOTHERSHIP_PORT:8080" \
-e SPAXEL_LOG_LEVEL=info \
-e TZ=UTC \
--tmpfs /data:size=100M \
"$MOTHERSHIP_IMAGE" >/dev/null
if [ $? -ne 0 ]; then
log_error "Failed to start mothership container"
exit 1
fi
log_info "Mothership container started: $MOTHERSHIP_CONTAINER"
# Step 2: Wait for /healthz to return {status:'ok'}
log_info "Step 2: Waiting for mothership to be healthy..."
start_time=$(date +%s)
while true; do
elapsed=$(($(date +%s) - start_time))
if [ $elapsed -ge $HEALTH_TIMEOUT ]; then
log_error "Health check timeout after ${HEALTH_TIMEOUT}s"
docker logs "$MOTHERSHIP_CONTAINER" --tail 50
exit 1
fi
health_response=$(http_get "http://localhost:$MOTHERSHIP_PORT/healthz" 1 0 2>/dev/null || echo "")
if [ -n "$health_response" ]; then
status=$(json_field "$health_response" ".status")
if [ "$status" = "ok" ]; then
log_info "Mothership is healthy (status: ok, uptime: $(json_field "$health_response" ".uptime_s")s)"
break
fi
fi
sleep 0.5
done
# Step 3: Check if PIN auth is enabled, setup if needed
log_info "Step 3: Checking auth setup..."
auth_status=$(http_get "http://localhost:$MOTHERSHIP_PORT/api/auth/status" 1 0 2>/dev/null || echo "{}")
pin_configured=$(json_field "$auth_status" ".pin_configured // false")
if [ "$pin_configured" = "false" ]; then
log_info "Setting up test PIN..."
setup_response=$(curl -sS -X POST \
-H "Content-Type: application/json" \
-d '{"pin":"0000"}' \
"http://localhost:$MOTHERSHIP_PORT/api/auth/setup" 2>/dev/null || echo "")
if [ -n "$setup_response" ]; then
ok=$(json_field "$setup_response" ".ok // false")
if [ "$ok" = "true" ]; then
log_info "Test PIN configured successfully"
else
log_warn "PIN setup response unexpected: $setup_response"
fi
fi
# Login with the PIN
log_info "Logging in with test PIN..."
login_response=$(curl -sS -X POST \
-H "Content-Type: application/json" \
-d '{"pin":"0000"}' \
-c /tmp/spaxel-e2e-cookies.txt \
"http://localhost:$MOTHERSHIP_PORT/api/auth/login" 2>/dev/null || echo "")
fi
# Step 4: Build and start simulator
log_info "Step 4: Starting CSI simulator..."
# Build simulator
cd /home/coding/spaxel/mothership
if ! go build -o /tmp/spaxel-sim ./cmd/sim 2>/dev/null; then
log_error "Failed to build simulator"
exit 1
fi
# Start simulator in background
/tmp/spaxel-sim \
--mothership "ws://localhost:$MOTHERSHIP_PORT/ws/node" \
--nodes "$SIM_NODES" \
--walkers "$SIM_WALKERS" \
--rate "$SIM_RATE" \
--duration "${SIM_DURATION}s" \
--ble \
--seed "$SIM_SEED" \
--show-frame-rate \
> /tmp/spaxel-sim.log 2>&1 &
SIM_PID=$!
if [ -z "$SIM_PID" ]; then
log_error "Failed to start simulator"
exit 1
fi
log_info "Simulator started (PID: $SIM_PID), will run for ${SIM_DURATION}s"
# Step 5: Assert during run (poll every 1s for up to SIM_DURATION)
log_info "Step 5: Asserting during simulation..."
assert_start=$(date +%s)
blob_detected=0
nodes_online=0
health_ok_during_run=0
health_check_count=0
health_check_passed=0
while true; do
elapsed=$(($(date +%s) - assert_start))
if [ $elapsed -ge $SIM_DURATION ]; then
break
fi
# Check /healthz - assert status=='ok' throughout entire run
health_response=$(http_get "http://localhost:$MOTHERSHIP_PORT/healthz" 1 0 2>/dev/null || echo "")
if [ -n "$health_response" ]; then
status=$(json_field "$health_response" ".status")
health_check_count=$((health_check_count + 1))
if [ "$status" = "ok" ]; then
health_ok_during_run=1
health_check_passed=$((health_check_passed + 1))
else
log_error "Health check failed during run: status=$status"
fi
fi
# Check /api/nodes for online nodes
nodes_response=$(http_get "http://localhost:$MOTHERSHIP_PORT/api/nodes" 1 0 2>/dev/null || echo "")
if [ -n "$nodes_response" ]; then
nodes_online=$(echo "$nodes_response" | jq '[.[] | select(.status=="online")] | length' 2>/dev/null || echo "0")
# Assert nodes_online == SIM_NODES within first 5 seconds
if [ $elapsed -le 5 ] && [ "$nodes_online" -ge "$SIM_NODES" ]; then
log_info "✓ All $SIM_NODES nodes online within first 5s (elapsed: ${elapsed}s)"
fi
fi
# Check for blobs via /api/events (detection events) after 5 seconds
if [ $elapsed -ge 5 ]; then
events_response=$(http_get "http://localhost:$MOTHERSHIP_PORT/api/events?type=detection&limit=10" 1 0 2>/dev/null || echo "")
if [ -n "$events_response" ]; then
event_count=$(echo "$events_response" | jq '.events | length' 2>/dev/null || echo "0")
if [ "$event_count" -gt 0 ]; then
blob_detected=1
if [ $elapsed -le 15 ]; then
log_info "✓ Blob detected within first 15s (found $event_count detection events at ${elapsed}s)"
fi
fi
fi
fi
sleep 1
done
# Step 6: Wait for simulator to complete
log_info "Waiting for simulator to complete..."
if ! wait $SIM_PID; then
log_error "Simulator exited with non-zero status"
cat /tmp/spaxel-sim.log
exit 1
fi
log_info "Simulator completed successfully"
# Step 7: Assert after run
log_info "Step 7: Asserting after simulation..."
# Assert health remained ok throughout run
if [ $health_check_passed -lt $((health_check_count * 95 / 100)) ]; then
log_error "Health check failed too often: $health_check_passed/$health_check_count passed"
exit 1
fi
log_info "✓ Health remained ok during run ($health_check_passed/$health_check_count checks passed)"
# Check /healthz still ok
health_response=$(http_get "http://localhost:$MOTHERSHIP_PORT/healthz" 5 1 2>/dev/null || echo "")
if [ -z "$health_response" ]; then
log_error "Health check failed after simulation"
exit 1
fi
status=$(json_field "$health_response" ".status")
if [ "$status" != "ok" ]; then
log_error "Health status not ok after simulation: $status"
exit 1
fi
log_info "✓ Health check passed after simulation"
# Check detection events were recorded
events_response=$(http_get "http://localhost:$MOTHERSHIP_PORT/api/events?type=detection&limit=100" 5 1 2>/dev/null || echo "")
if [ -z "$events_response" ]; then
log_error "Failed to get events after simulation"
exit 1
fi
event_count=$(echo "$events_response" | jq '.events | length' 2>/dev/null || echo "0")
if [ "$event_count" -lt 1 ]; then
log_error "No detection events recorded after simulation"
log_error "Events response: $events_response"
exit 1
fi
log_info "✓ At least 1 detection event recorded (found $event_count events)"
# Check simulator output for frame rate
frame_count=$(grep -o "sent [0-9]* frames" /tmp/spaxel-sim.log | tail -1 | grep -o "[0-9]*" || echo "0")
if [ "$frame_count" -gt 0 ]; then
expected_frames=$((SIM_NODES * SIM_RATE * SIM_DURATION))
# Sum up all frame counts from the log
actual_frames=$(grep "sent.*frames" /tmp/spaxel-sim.log | awk '{for(i=1;i<=NF;i++) if($i~/^[0-9]+$/) sum+=$i} END {print sum+0}' || echo "0")
if [ "$actual_frames" -gt 0 ]; then
frame_rate_ratio=$((actual_frames * 100 / expected_frames))
if [ $frame_rate_ratio -lt 80 ]; then
log_error "Frame rate dropped more than 20% (got $frame_rate_ratio% of expected)"
log_error "Expected: $expected_frames frames, Got: $actual_frames frames"
exit 1
fi
log_info "✓ Frame rate acceptable ($frame_rate_ratio% of expected $expected_frames frames)"
fi
fi
# Assert blob was detected within 15s
if [ $blob_detected -eq 0 ]; then
log_error "No blob detected within first 15s"
exit 1
fi
log_info "✓ Blob was detected during simulation"
# Summary
log_info ""
log_info "=== E2E Test Summary ==="
log_info "✓ Mothership started and became healthy"
log_info "✓ Simulator ran for ${SIM_DURATION}s with $SIM_NODES nodes, $SIM_WALKERS walkers"
log_info "✓ Health remained ok during run ($health_check_passed/$health_check_count checks)"
log_info "✓ Nodes came online ($nodes_online of $SIM_NODES)"
log_info "✓ Detection events recorded ($event_count events)"
log_info "✓ Blob detected within 15s"
log_info "✓ Frame rate acceptable"
log_info ""
total_time=$(($(date +%s) - start_time))
log_info "Total test time: ${total_time}s (target: <${TEST_TIMEOUT}s)"
if [ $total_time -ge $TEST_TIMEOUT ]; then
log_warn "Test exceeded target time of ${TEST_TIMEOUT}s"
fi
log_info "All assertions passed!"
exit 0