304879d32a
Add comprehensive chaos testing infrastructure for Miroir failure scenarios: - **TestCluster** harness with chaos helpers: - `kill_meili()` / `restart_meili()` for node failure simulation - `apply_netem()` / `remove_netem()` for network delay injection - `kill_miroir()` / `restart_miroir()` for orchestrator failure - Docker-compose stack lifecycle management - **6 chaos test scenarios** (all marked `#[ignore]`): 1. Kill 1 of 3 nodes (RF=2) → continuous search, no degraded header 2. Kill 2 of 3 nodes (RF=2) → 503 or partial results with degraded header 3. Kill 1 of 2 Miroir replicas → zero client-visible downtime 4. tc netem 500ms delay → searches slow but succeed, no errors 5. Restart killed node → Miroir detects recovery within health check interval 6. Kill node mid-rebalance → rebalancer pauses, resumes on recovery - **Runbooks** in `tests/chaos/runbooks/scenario*.md`: - Manual reproduction steps - Expected observables (metrics, headers, errors) - Recovery procedures - HA vs single-instance differences - Operator notes and common causes - **Updated docker-compose files**: - Added `CAP_NET_ADMIN` to all Meilisearch containers for tc netem support Tests are slow (30+ seconds each) and require docker-compose. Run with: cargo test --test chaos -- --ignored --test-threads=1 Closes: miroir-89x.4
128 lines
3.4 KiB
YAML
128 lines
3.4 KiB
YAML
# Miroir development stack — 3 Meilisearch nodes + 1 Miroir orchestrator
|
|
# Quick start: docker compose -f examples/docker-compose-dev.yml up -d
|
|
|
|
services:
|
|
# Meilisearch node 0 (shard replica group 0)
|
|
meili-0:
|
|
image: getmeili/meilisearch:v1.37.0
|
|
container_name: miroir-meili-0
|
|
cap_add:
|
|
- NET_ADMIN
|
|
environment:
|
|
- MEILI_ENV=development
|
|
- MEILI_MASTER_KEY=dev-node-key
|
|
- MEILI_NO_ANALYTICS=true
|
|
ports:
|
|
- "7701:7700"
|
|
volumes:
|
|
- meili-0-data:/meili_data
|
|
healthcheck:
|
|
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:7700/health"]
|
|
interval: 5s
|
|
timeout: 2s
|
|
retries: 3
|
|
|
|
# Meilisearch node 1 (shard replica group 0)
|
|
meili-1:
|
|
image: getmeili/meilisearch:v1.37.0
|
|
container_name: miroir-meili-1
|
|
cap_add:
|
|
- NET_ADMIN
|
|
environment:
|
|
- MEILI_ENV=development
|
|
- MEILI_MASTER_KEY=dev-node-key
|
|
- MEILI_NO_ANALYTICS=true
|
|
ports:
|
|
- "7702:7700"
|
|
volumes:
|
|
- meili-1-data:/meili_data
|
|
healthcheck:
|
|
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:7700/health"]
|
|
interval: 5s
|
|
timeout: 2s
|
|
retries: 3
|
|
|
|
# Meilisearch node 2 (shard replica group 0)
|
|
meili-2:
|
|
image: getmeili/meilisearch:v1.37.0
|
|
container_name: miroir-meili-2
|
|
cap_add:
|
|
- NET_ADMIN
|
|
environment:
|
|
- MEILI_ENV=development
|
|
- MEILI_MASTER_KEY=dev-node-key
|
|
- MEILI_NO_ANALYTICS=true
|
|
ports:
|
|
- "7703:7700"
|
|
volumes:
|
|
- meili-2-data:/meili_data
|
|
healthcheck:
|
|
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:7700/health"]
|
|
interval: 5s
|
|
timeout: 2s
|
|
retries: 3
|
|
|
|
# Miroir orchestrator
|
|
miroir:
|
|
build:
|
|
context: ..
|
|
dockerfile: Dockerfile
|
|
image: miroir-dev:latest
|
|
container_name: miroir-orchestrator
|
|
environment:
|
|
- MIROIR_MASTER_KEY=dev-key
|
|
- MIROIR_NODE_MASTER_KEY=dev-node-key
|
|
ports:
|
|
- "7700:7700"
|
|
volumes:
|
|
- ../examples/dev-config.yaml:/etc/miroir/config.yaml:ro
|
|
- miroir-data:/data
|
|
depends_on:
|
|
meili-0:
|
|
condition: service_healthy
|
|
meili-1:
|
|
condition: service_healthy
|
|
meili-2:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:7700/health"]
|
|
interval: 5s
|
|
timeout: 2s
|
|
retries: 3
|
|
|
|
# Standalone Meilisearch for API compatibility testing
|
|
# This instance is NOT managed by Miroir — it's used for comparison testing
|
|
meili-standalone:
|
|
image: getmeili/meilisearch:v1.37.0
|
|
container_name: miroir-meili-standalone
|
|
environment:
|
|
- MEILI_ENV=development
|
|
- MEILI_MASTER_KEY=dev-node-key
|
|
- MEILI_NO_ANALYTICS=true
|
|
ports:
|
|
- "7704:7700"
|
|
volumes:
|
|
- meili-standalone-data:/meili_data
|
|
healthcheck:
|
|
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:7700/health"]
|
|
interval: 5s
|
|
timeout: 2s
|
|
retries: 3
|
|
|
|
# Optional: Redis for multi-replica Miroir deployments
|
|
# Uncomment this service and set task_store.backend=redis in dev-config.yaml
|
|
# redis:
|
|
# image: redis:7-alpine
|
|
# container_name: miroir-redis
|
|
# ports:
|
|
# - "6379:6379"
|
|
# volumes:
|
|
# - redis-data:/data
|
|
|
|
volumes:
|
|
meili-0-data:
|
|
meili-1-data:
|
|
meili-2-data:
|
|
meili-standalone-data:
|
|
miroir-data:
|
|
# redis-data:
|