#!/bin/bash # End-to-end integration test harness for Spaxel # Starts the mothership, runs the CSI simulator, and asserts on behavior set -eo pipefail # Get the script directory and project root SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" MOTHERSHIP_DIR="$PROJECT_ROOT/mothership" # Configuration MOTHERSHIP_IMAGE="${MOTHERSHIP_IMAGE:-spaxel-e2e:test}" LOCAL_BUILD="${LOCAL_BUILD:-false}" # Set to "true" to use local build instead of Docker MOTHERSHIP_CONTAINER="spaxel-e2e-test" MOTHERSHIP_PORT=8080 HEALTH_TIMEOUT=15 SIM_DURATION=30 SIM_NODES=4 SIM_WALKERS=2 SIM_RATE=20 SIM_SEED=42 TEST_TIMEOUT=90 # Initialize PIDs SIM_PID="" MOTHERSHIP_PID="" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # No Color # Logging functions log_info() { echo -e "${GREEN}[INFO]${NC} $1" } log_warn() { echo -e "${YELLOW}[WARN]${NC} $1" } log_error() { echo -e "${RED}[ERROR]${NC} $1" } # Cleanup function cleanup() { local exit_code=$? log_info "Cleaning up..." # Stop simulator if running if [ -n "$SIM_PID" ]; then kill $SIM_PID 2>/dev/null || true wait $SIM_PID 2>/dev/null || true fi # Stop mothership (container or local) if [ "$LOCAL_BUILD" = "true" ]; then if [ -n "$MOTHERSHIP_PID" ]; then kill $MOTHERSHIP_PID 2>/dev/null || true wait $MOTHERSHIP_PID 2>/dev/null || true fi # Clean up temp data directory if [ -n "$TEST_DATA_DIR" ] && [ -d "$TEST_DATA_DIR" ]; then rm -rf "$TEST_DATA_DIR" fi else # Stop and remove mothership container docker stop "$MOTHERSHIP_CONTAINER" 2>/dev/null || true docker rm "$MOTHERSHIP_CONTAINER" 2>/dev/null || true fi if [ $exit_code -eq 0 ]; then log_info "All tests passed!" else log_error "Tests failed with exit code $exit_code" fi exit $exit_code } trap cleanup EXIT INT TERM # Helper function for HTTP requests with timeout http_get() { local url="$1" local max_attempts="${2:-1}" local wait="${3:-0}" for i in $(seq 1 "$max_attempts"); do if curl -sSf --max-time 5 "$url" 2>/dev/null; then return 0 fi if [ $i -lt $max_attempts ]; then sleep "$wait" fi done return 1 } # Helper function to check JSON field json_field() { local json="$1" local field="$2" echo "$json" | jq -r "$field // empty" } # Step 1: Start mothership container or local build test_start_time=$(date +%s) if [ "$LOCAL_BUILD" = "true" ]; then log_info "Step 1: Starting local mothership build..." # Build mothership if needed if [ ! -f /tmp/spaxel-mothership-test ]; then log_info "Building mothership..." cd "$MOTHERSHIP_DIR" if ! go build -o /tmp/spaxel-mothership-test ./cmd/mothership; then log_error "Failed to build mothership" exit 1 fi fi # Create temp data directory TEST_DATA_DIR=$(mktemp -d -t spaxel-e2e-data-XXXXXX) # Start local mothership in background with environment variables SPAXEL_BIND_ADDR="127.0.0.1:$MOTHERSHIP_PORT" \ SPAXEL_DATA_DIR="$TEST_DATA_DIR" \ SPAXEL_LOG_LEVEL=info \ TZ=UTC \ /tmp/spaxel-mothership-test > /tmp/spaxel-mothership.log 2>&1 & MOTHERSHIP_PID=$! log_info "Local mothership started (PID: $MOTHERSHIP_PID, data: $TEST_DATA_DIR)" else log_info "Step 1: Starting mothership container..." # Check if container is already running and remove it docker rm -f "$MOTHERSHIP_CONTAINER" 2>/dev/null || true # Run mothership container docker run -d \ --name "$MOTHERSHIP_CONTAINER" \ -p "$MOTHERSHIP_PORT:8080" \ -e SPAXEL_LOG_LEVEL=info \ -e TZ=UTC \ --tmpfs /data:size=100M \ "$MOTHERSHIP_IMAGE" >/dev/null if [ $? -ne 0 ]; then log_error "Failed to start mothership container" exit 1 fi log_info "Mothership container started: $MOTHERSHIP_CONTAINER" fi # Step 2: Wait for /healthz to return {status:'ok'} log_info "Step 2: Waiting for mothership to be healthy..." start_time=$(date +%s) while true; do elapsed=$(($(date +%s) - start_time)) if [ $elapsed -ge $HEALTH_TIMEOUT ]; then log_error "Health check timeout after ${HEALTH_TIMEOUT}s" if [ "$LOCAL_BUILD" = "true" ]; then cat /tmp/spaxel-mothership.log | tail -50 else docker logs "$MOTHERSHIP_CONTAINER" --tail 50 fi exit 1 fi health_response=$(http_get "http://localhost:$MOTHERSHIP_PORT/healthz" 1 0 2>/dev/null || echo "") if [ -n "$health_response" ]; then status=$(echo "$health_response" | jq -r '.status // empty' 2>/dev/null || echo "") if [ "$status" = "ok" ]; then uptime=$(echo "$health_response" | jq -r '.uptime_s // 0' 2>/dev/null || echo "0") log_info "Mothership is healthy (status: ok, uptime: ${uptime}s)" break fi fi sleep 0.5 done # Step 3: Check if PIN auth is enabled, setup if needed log_info "Step 3: Checking auth setup..." # Try to check auth status, but continue even if endpoint doesn't exist http_code=$(curl -sS -o /dev/null -w "%{http_code}" "http://localhost:$MOTHERSHIP_PORT/api/auth/status" 2>/dev/null || echo "000") # Only proceed with auth setup if endpoint exists (HTTP 200, not 404) if [ "$http_code" = "200" ]; then auth_status=$(curl -sS "http://localhost:$MOTHERSHIP_PORT/api/auth/status" 2>/dev/null || echo "") pin_configured=$(json_field "$auth_status" ".pin_configured // false") if [ "$pin_configured" = "false" ]; then log_info "Setting up test PIN..." setup_response=$(curl -sS -X POST \ -H "Content-Type: application/json" \ -d '{"pin":"0000"}' \ "http://localhost:$MOTHERSHIP_PORT/api/auth/setup" 2>/dev/null || echo "") if [ -n "$setup_response" ]; then ok=$(json_field "$setup_response" ".ok // false") if [ "$ok" = "true" ]; then log_info "Test PIN configured successfully" # Login with the PIN log_info "Logging in with test PIN..." login_response=$(curl -sS -X POST \ -H "Content-Type: application/json" \ -d '{"pin":"0000"}' \ -c /tmp/spaxel-e2e-cookies.txt \ "http://localhost:$MOTHERSHIP_PORT/api/auth/login" 2>/dev/null || echo "") else log_warn "PIN setup response unexpected: $setup_response" fi fi else log_info "Auth already configured, skipping setup" fi else log_info "Auth endpoint not available (HTTP $http_code), running without auth..." fi # Step 4: Build and start simulator log_info "Step 4: Starting CSI simulator..." # Build simulator using Docker (since go may not be available on host) if [ ! -f /tmp/spaxel-sim ]; then log_info "Building simulator with Docker..." docker run --rm \ -v "$PROJECT_ROOT/mothership:/src" \ -v /tmp:/out \ -w /src \ golang:1.25-bookworm \ sh -c "go build -o /out/spaxel-sim ./cmd/sim" fi if [ ! -f /tmp/spaxel-sim ]; then log_error "Failed to build simulator" exit 1 fi # Start simulator in background /tmp/spaxel-sim \ --mothership "ws://localhost:$MOTHERSHIP_PORT/ws/node" \ --nodes "$SIM_NODES" \ --walkers "$SIM_WALKERS" \ --rate "$SIM_RATE" \ --duration "${SIM_DURATION}s" \ --ble \ --seed "$SIM_SEED" \ --show-frame-rate \ > /tmp/spaxel-sim.log 2>&1 & SIM_PID=$! if [ -z "$SIM_PID" ]; then log_error "Failed to start simulator" exit 1 fi log_info "Simulator started (PID: $SIM_PID), will run for ${SIM_DURATION}s" # Step 5: Assert during run (poll every 1s for up to SIM_DURATION) log_info "Step 5: Asserting during simulation..." assert_start=$(date +%s) blob_detected=0 nodes_online=0 health_ok_during_run=0 health_check_count=0 health_check_passed=0 while true; do elapsed=$(($(date +%s) - assert_start)) if [ $elapsed -ge $SIM_DURATION ]; then break fi # Check /healthz - assert status=='ok' throughout entire run health_response=$(http_get "http://localhost:$MOTHERSHIP_PORT/healthz" 1 0 2>/dev/null || echo "") if [ -n "$health_response" ]; then status=$(json_field "$health_response" ".status") health_check_count=$((health_check_count + 1)) if [ "$status" = "ok" ]; then health_ok_during_run=1 health_check_passed=$((health_check_passed + 1)) else log_error "Health check failed during run: status=$status" fi fi # Check /api/nodes for online nodes nodes_response=$(http_get "http://localhost:$MOTHERSHIP_PORT/api/nodes" 1 0 2>/dev/null || echo "") if [ -n "$nodes_response" ]; then # Count nodes with status "online" nodes_online=$(echo "$nodes_response" | jq '[.[] | select(.status=="online")] | length' 2>/dev/null || echo "0") # Assert nodes_online == SIM_NODES within first 5 seconds if [ $elapsed -le 5 ] && [ "$nodes_online" -ge "$SIM_NODES" ]; then log_info "✓ All $SIM_NODES nodes online within first 5s (elapsed: ${elapsed}s)" fi fi # Check for blobs via /api/blobs after 5 seconds if [ $elapsed -ge 5 ]; then blobs_response=$(http_get "http://localhost:$MOTHERSHIP_PORT/api/blobs" 1 0 2>/dev/null || echo "") if [ -n "$blobs_response" ]; then blob_count=$(echo "$blobs_response" | jq 'length' 2>/dev/null || echo "0") if [ "$blob_count" -gt 0 ]; then blob_detected=1 if [ $elapsed -le 15 ]; then log_info "✓ Blob detected within first 15s (found $blob_count blobs at ${elapsed}s)" fi fi fi fi sleep 1 done # Step 6: Wait for simulator to complete log_info "Waiting for simulator to complete..." if ! wait $SIM_PID; then log_error "Simulator exited with non-zero status" cat /tmp/spaxel-sim.log exit 1 fi log_info "Simulator completed successfully" # Step 7: Assert after run log_info "Step 7: Asserting after simulation..." # Assert health remained ok throughout run if [ $health_check_passed -lt $((health_check_count * 95 / 100)) ]; then log_error "Health check failed too often: $health_check_passed/$health_check_count passed" exit 1 fi log_info "✓ Health remained ok during run ($health_check_passed/$health_check_count checks passed)" # Check /healthz still ok health_response=$(http_get "http://localhost:$MOTHERSHIP_PORT/healthz" 5 1 2>/dev/null || echo "") if [ -z "$health_response" ]; then log_error "Health check failed after simulation" exit 1 fi status=$(json_field "$health_response" ".status") if [ "$status" != "ok" ]; then log_error "Health status not ok after simulation: $status" exit 1 fi log_info "✓ Health check passed after simulation" # Check detection events were recorded events_response=$(http_get "http://localhost:$MOTHERSHIP_PORT/api/events?limit=100" 5 1 2>/dev/null || echo "") if [ -z "$events_response" ]; then log_error "Failed to get events after simulation" exit 1 fi # Try both .events format and direct array format event_count=$(echo "$events_response" | jq '.events | length' 2>/dev/null || echo "0") if [ "$event_count" = "0" ]; then event_count=$(echo "$events_response" | jq 'length' 2>/dev/null || echo "0") fi if [ "$event_count" -lt 1 ]; then log_error "No detection events recorded after simulation" log_error "Events response: $events_response" exit 1 fi log_info "✓ At least 1 detection event recorded (found $event_count events)" # Check simulator output for frame rate # Format: "[STATS] Node AA:BB:CC:DD:XX:00: sent 123 frames" frame_count=$(grep -o "sent [0-9]* frames" /tmp/spaxel-sim.log | tail -1 | grep -o "[0-9]*" || echo "0") if [ "$frame_count" -gt 0 ]; then expected_frames=$((SIM_NODES * SIM_RATE * SIM_DURATION)) # Sum up all frame counts from the log using more precise pattern actual_frames=$(grep -o "sent [0-9]* frames" /tmp/spaxel-sim.log | grep -o "[0-9]*" | awk '{sum+=$1} END {print sum+0}' || echo "0") if [ "$actual_frames" -gt 0 ]; then frame_rate_ratio=$((actual_frames * 100 / expected_frames)) if [ $frame_rate_ratio -lt 80 ]; then log_error "Frame rate dropped more than 20% (got $frame_rate_ratio% of expected)" log_error "Expected: $expected_frames frames, Got: $actual_frames frames" exit 1 fi log_info "✓ Frame rate acceptable ($frame_rate_ratio% of expected $expected_frames frames)" fi fi # Assert blob was detected within 15s if [ $blob_detected -eq 0 ]; then log_error "No blob detected within first 15s" exit 1 fi log_info "✓ Blob was detected during simulation" # Summary log_info "" log_info "=== E2E Test Summary ===" log_info "✓ Mothership started and became healthy" log_info "✓ Simulator ran for ${SIM_DURATION}s with $SIM_NODES nodes, $SIM_WALKERS walkers" log_info "✓ Health remained ok during run ($health_check_passed/$health_check_count checks)" log_info "✓ Nodes came online ($nodes_online of $SIM_NODES)" log_info "✓ Detection events recorded ($event_count events)" log_info "✓ Blob detected within 15s" log_info "✓ Frame rate acceptable" log_info "" total_time=$(($(date +%s) - start_time)) log_info "Total test time: ${total_time}s (target: <${TEST_TIMEOUT}s)" if [ $total_time -ge $TEST_TIMEOUT ]; then log_warn "Test exceeded target time of ${TEST_TIMEOUT}s" fi log_info "All assertions passed!" exit 0