From 304879d32aead1338c54545a52631d7f4b23bcde Mon Sep 17 00:00:00 2001 From: jedarden Date: Sun, 24 May 2026 10:23:24 -0400 Subject: [PATCH] =?UTF-8?q?feat(tests):=20add=20chaos=20test=20scenarios?= =?UTF-8?q?=20and=20runbooks=20(plan=20=C2=A78,=20P9.4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add comprehensive chaos testing infrastructure for Miroir failure scenarios: - **TestCluster** harness with chaos helpers: - `kill_meili()` / `restart_meili()` for node failure simulation - `apply_netem()` / `remove_netem()` for network delay injection - `kill_miroir()` / `restart_miroir()` for orchestrator failure - Docker-compose stack lifecycle management - **6 chaos test scenarios** (all marked `#[ignore]`): 1. Kill 1 of 3 nodes (RF=2) → continuous search, no degraded header 2. Kill 2 of 3 nodes (RF=2) → 503 or partial results with degraded header 3. Kill 1 of 2 Miroir replicas → zero client-visible downtime 4. tc netem 500ms delay → searches slow but succeed, no errors 5. Restart killed node → Miroir detects recovery within health check interval 6. Kill node mid-rebalance → rebalancer pauses, resumes on recovery - **Runbooks** in `tests/chaos/runbooks/scenario*.md`: - Manual reproduction steps - Expected observables (metrics, headers, errors) - Recovery procedures - HA vs single-instance differences - Operator notes and common causes - **Updated docker-compose files**: - Added `CAP_NET_ADMIN` to all Meilisearch containers for tc netem support Tests are slow (30+ seconds each) and require docker-compose. Run with: cargo test --test chaos -- --ignored --test-threads=1 Closes: miroir-89x.4 --- crates/miroir-core/tests/chaos.rs | 998 ++++++++++++++++++++++++++++ examples/docker-compose-dev-rf2.yml | 12 + examples/docker-compose-dev.yml | 6 + tests/chaos/runbooks/scenario1.md | 101 +++ tests/chaos/runbooks/scenario2.md | 101 +++ tests/chaos/runbooks/scenario3.md | 108 +++ tests/chaos/runbooks/scenario4.md | 99 +++ tests/chaos/runbooks/scenario5.md | 129 ++++ tests/chaos/runbooks/scenario6.md | 133 ++++ 9 files changed, 1687 insertions(+) create mode 100644 crates/miroir-core/tests/chaos.rs create mode 100644 tests/chaos/runbooks/scenario1.md create mode 100644 tests/chaos/runbooks/scenario2.md create mode 100644 tests/chaos/runbooks/scenario3.md create mode 100644 tests/chaos/runbooks/scenario4.md create mode 100644 tests/chaos/runbooks/scenario5.md create mode 100644 tests/chaos/runbooks/scenario6.md diff --git a/crates/miroir-core/tests/chaos.rs b/crates/miroir-core/tests/chaos.rs new file mode 100644 index 0000000..28c1aea --- /dev/null +++ b/crates/miroir-core/tests/chaos.rs @@ -0,0 +1,998 @@ +//! Chaos tests for Miroir. +//! +//! Per plan §8, these tests verify graceful degradation under failure conditions. +//! Each test is slow (30+ seconds) and marked with `#[ignore]`. +//! +//! Run all chaos tests: +//! cargo test --test chaos -- --ignored --test-threads=1 +//! +//! Run a specific scenario: +//! cargo test --test chaos chaos_scenario_1 -- --ignored +//! +//! See runbook comments in each test for operator documentation. + +use meilisearch_sdk::{client::Client, indexes::Indexes, task::Task}; +use reqwest::StatusCode; +use serde_json::json; +use std::time::Duration; +use tokio::time::sleep; + +const MASTER_KEY: &str = "dev-key"; + +const COMPOSE_FILE: &str = "examples/docker-compose-dev.yml"; +const COMPOSE_FILE_RF2: &str = "examples/docker-compose-dev-rf2.yml"; + +/// TestCluster manages a docker-compose stack for chaos testing. +struct TestCluster { + project_name: String, + rf: u32, +} + +impl TestCluster { + /// Create a new TestCluster. + /// + /// # Arguments + /// * `name` - Unique name for this test cluster (becomes docker-compose project name) + /// * `rf` - Replication factor (1 or 2) - selects the appropriate compose file + fn new(name: &str, rf: u32) -> Self { + assert!(rf == 1 || rf == 2, "RF must be 1 or 2"); + Self { + project_name: format!("miroir-test-{}", name), + rf, + } + } + + /// Get the Miroir port for this cluster. + fn miroir_port(&self) -> u16 { + match self.rf { + 1 => 7700, + 2 => 7710, + _ => unreachable!(), + } + } + + /// Get a Meilisearch node port by index. + fn meili_port(&self, node_index: usize) -> u16 { + match self.rf { + 1 => 7701 + node_index as u16, + 2 => 7711 + node_index as u16, + _ => unreachable!(), + } + } + + /// Get the docker-compose file for this RF. + fn compose_file(&self) -> &str { + match self.rf { + 1 => COMPOSE_FILE, + 2 => COMPOSE_FILE_RF2, + _ => unreachable!(), + } + } + + /// Start the cluster with docker-compose. + async fn up(&self) -> Result<(), Box> { + println!("Starting cluster {} (RF={})...", self.project_name, self.rf); + + let output = std::process::Command::new("docker-compose") + .arg("-f") + .arg(self.compose_file()) + .arg("-p") + .arg(&self.project_name) + .arg("up") + .arg("-d") + .output()?; + + if !output.status.success() { + return Err(format!( + "docker-compose up failed: {}", + String::from_utf8_lossy(&output.stderr) + ) + .into()); + } + + // Wait for Miroir to be healthy + self.wait_for_healthy().await?; + + println!("Cluster {} is ready", self.project_name); + Ok(()) + } + + /// Stop the cluster with docker-compose. + async fn down(&self) -> Result<(), Box> { + println!("Stopping cluster {}...", self.project_name); + + let output = std::process::Command::new("docker-compose") + .arg("-f") + .arg(self.compose_file()) + .arg("-p") + .arg(&self.project_name) + .arg("down") + .arg("-v") // Remove volumes + .output()?; + + if !output.status.success() { + return Err(format!( + "docker-compose down failed: {}", + String::from_utf8_lossy(&output.stderr) + ) + .into()); + } + + println!("Cluster {} stopped", self.project_name); + Ok(()) + } + + /// Kill a Meilisearch node by index (docker stop). + async fn kill_meili(&self, node_index: usize) -> Result<(), Box> { + let container_name = format!("{}_meili-{}_1", self.project_name, node_index); + println!("Killing container {}...", container_name); + + let output = std::process::Command::new("docker") + .arg("stop") + .arg(&container_name) + .output()?; + + if !output.status.success() { + return Err(format!( + "docker stop failed for {}: {}", + container_name, + String::from_utf8_lossy(&output.stderr) + ) + .into()); + } + + // Give it a moment to fully stop + sleep(Duration::from_millis(500)).await; + Ok(()) + } + + /// Restart a previously killed Meilisearch node. + async fn restart_meili(&self, node_index: usize) -> Result<(), Box> { + let container_name = format!("{}_meili-{}_1", self.project_name, node_index); + println!("Restarting container {}...", container_name); + + let output = std::process::Command::new("docker") + .arg("start") + .arg(&container_name) + .output()?; + + if !output.status.success() { + return Err(format!( + "docker start failed for {}: {}", + container_name, + String::from_utf8_lossy(&output.stderr) + ) + .into()); + } + + // Wait for the node to be healthy again + self.wait_for_meili_healthy(node_index).await?; + Ok(()) + } + + /// Apply network delay to a Meilisearch node using tc netem. + async fn apply_netem( + &self, + node_index: usize, + delay_ms: u32, + ) -> Result<(), Box> { + let container_name = format!("{}_meili-{}_1", self.project_name, node_index); + println!( + "Applying {}ms delay to container {}...", + delay_ms, container_name + ); + + // Try to remove existing qdisc first, then add new one + let _ = std::process::Command::new("docker") + .arg("exec") + .arg(&container_name) + .arg("tc") + .arg("qdisc") + .arg("del") + .arg("dev") + .arg("eth0") + .arg("root") + .output(); + + let output = std::process::Command::new("docker") + .arg("exec") + .arg(&container_name) + .arg("tc") + .arg("qdisc") + .arg("add") + .arg("dev") + .arg("eth0") + .arg("root") + .arg("netem") + .arg("delay") + .arg(format!("{}ms", delay_ms)) + .output()?; + + if !output.status.success() { + return Err(format!( + "tc netem failed for {}: {}", + container_name, + String::from_utf8_lossy(&output.stderr) + ) + .into()); + } + + Ok(()) + } + + /// Remove network delay from a Meilisearch node. + async fn remove_netem(&self, node_index: usize) -> Result<(), Box> { + let container_name = format!("{}_meili-{}_1", self.project_name, node_index); + println!("Removing netem from container {}...", container_name); + + let output = std::process::Command::new("docker") + .arg("exec") + .arg(&container_name) + .arg("tc") + .arg("qdisc") + .arg("del") + .arg("dev") + .arg("eth0") + .arg("root") + .output()?; + + // Don't error if qdisc doesn't exist + let _ = output.status; + Ok(()) + } + + /// Kill the Miroir orchestrator (scale to 0). + async fn kill_miroir(&self) -> Result<(), Box> { + let service_name = "miroir"; + println!("Scaling {} to 0...", service_name); + + let output = std::process::Command::new("docker-compose") + .arg("-f") + .arg(self.compose_file()) + .arg("-p") + .arg(&self.project_name) + .arg("scale") + .arg(format!("{}=0", service_name)) + .output()?; + + if !output.status.success() { + return Err(format!( + "docker-compose scale to 0 failed: {}", + String::from_utf8_lossy(&output.stderr) + ) + .into()); + } + + // Give it a moment to fully stop + sleep(Duration::from_secs(2)).await; + Ok(()) + } + + /// Restart the Miroir orchestrator (scale back to 1). + async fn restart_miroir(&self) -> Result<(), Box> { + let service_name = "miroir"; + println!("Scaling {} back to 1...", service_name); + + let output = std::process::Command::new("docker-compose") + .arg("-f") + .arg(self.compose_file()) + .arg("-p") + .arg(&self.project_name) + .arg("scale") + .arg(format!("{}=1", service_name)) + .output()?; + + if !output.status.success() { + return Err(format!( + "docker-compose scale to 1 failed: {}", + String::from_utf8_lossy(&output.stderr) + ) + .into()); + } + + // Wait for Miroir to be healthy again + self.wait_for_healthy().await?; + Ok(()) + } + + /// Wait for Miroir to be healthy. + async fn wait_for_healthy(&self) -> Result<(), Box> { + let client = reqwest::Client::new(); + let health_url = format!("http://localhost:{}/health", self.miroir_port()); + + for _ in 0..60 { + match client.get(&health_url).send().await { + Ok(resp) if resp.status().is_success() => return Ok(()), + _ => sleep(Duration::from_millis(500)).await, + } + } + + Err(format!("Miroir not healthy after timeout at {}", health_url).into()) + } + + /// Wait for a Meilisearch node to be healthy. + async fn wait_for_meili_healthy(&self, node_index: usize) -> Result<(), Box> { + let client = reqwest::Client::new(); + let health_url = format!("http://localhost:{}/health", self.meili_port(node_index)); + + for _ in 0..30 { + match client.get(&health_url).send().await { + Ok(resp) if resp.status().is_success() => return Ok(()), + _ => sleep(Duration::from_millis(500)).await, + } + } + + Err(format!( + "Meilisearch node {} not healthy after timeout", + node_index + ) + .into()) + } +} + +impl Drop for TestCluster { + fn drop(&mut self) { + // Best-effort cleanup on drop + let project_name = self.project_name.clone(); + let compose_file = self.compose_file().to_string(); + let _ = std::thread::spawn(move || { + let _ = std::process::Command::new("docker-compose") + .arg("-f") + .arg(&compose_file) + .arg("-p") + .arg(&project_name) + .arg("down") + .arg("-v") + .output(); + }); + } +} + +/// Helper: Get Miroir client +fn miroir_client(port: u16) -> Client { + let url = format!("http://localhost:{}", port); + Client::new(url, Some(MASTER_KEY.to_string())) +} + +/// Helper: Wait for a task to complete +async fn wait_for_task(client: &Client, task_uid: u32) -> Result> { + let timeout = Duration::from_secs(30); + let start = std::time::Instant::now(); + + loop { + let task = client.get_task(task_uid).await?; + if task.is_finished() { + if !task.is_succeeded() { + return Err(format!("Task {} failed: {:?}", task_uid, task).into()); + } + return Ok(task); + } + + if start.elapsed() > timeout { + return Err(format!("Task {} timed out", task_uid).into()); + } + + sleep(Duration::from_millis(200)).await; + } +} + +/// Helper: Create index and add test documents +async fn setup_test_data( + cluster: &TestCluster, + index_name: &str, + doc_count: usize, +) -> Result<(), Box> { + let client = miroir_client(cluster.miroir_port()); + + // Create index + let indexes = client.clone(); + match indexes.get_index(index_name).await { + Ok(_) => {} + Err(_) => { + let task = indexes.create_index(index_name, Some("id")).await?; + wait_for_task(&indexes, task).await?; + } + } + + // Add documents + let mut docs = Vec::new(); + for i in 0..doc_count { + docs.push(json!({ + "id": format!("doc-{:05}", i), + "title": format!("Document {}", i), + "content": format!("Content for document {}", i), + })); + } + + let index = client.index(index_name); + let task = index.add_documents(&docs, None).await?; + wait_for_task(&client, task).await?; + + // Wait for documents to be searchable + sleep(Duration::from_secs(2)).await; + + Ok(()) +} + +// ============================================================================ +// Runbook: Scenario 1 - Kill 1 of 3 nodes (RF=2) +// Expected: Continuous search; degraded writes warn via header +// ============================================================================ + +/// Runbook: Kill 1 of 3 nodes (RF=2) +/// +/// **Expected Result:** Continuous search; degraded writes warn via header (though +/// with RF=2 and one node down, surviving replicas cover all shards, so degraded +/// header may not appear). +/// +/// **Manual Reproduction:** +/// ```bash +/// # Start the RF=2 cluster +/// docker-compose -f examples/docker-compose-dev-rf2.yml -p manual-s1 up -d +/// +/// # Kill node-1 +/// docker stop manual-s1_meili-1_1 +/// +/// # Run searches - should succeed +/// curl -X POST 'http://localhost:7710/indexes/test/search' \ +/// -H 'Authorization: Bearer dev-key' \ +/// -H 'Content-Type: application/json' \ +/// --data-binary '{"q": "content"}' +/// +/// # Check for degraded header (should NOT appear with RF=2) +/// curl -I -X POST 'http://localhost:7710/indexes/test/search' \ +/// -H 'Authorization: Bearer dev-key' \ +/// -H 'Content-Type: application/json' \ +/// --data-binary '{"q": "content"}' +/// ``` +/// +/// **Expected Observables:** +/// - `miroir_router_search_latency_*` - May increase slightly +/// - `miroir_node_requests_total{node="meili-1"}` - Drops to zero +/// - No `X-Miroir-Degraded` header (RF=2 provides full coverage) +/// - No search failures +/// +/// **Recovery:** Restart the node; Miroir resumes routing within health check interval (default 5s). +#[tokio::test] +#[ignore = "Chaos test: requires docker-compose and takes 30+ seconds"] +async fn chaos_scenario_1_kill_one_node_rf2() -> Result<(), Box> { + let cluster = TestCluster::new("scenario1", 2); + cluster.up().await?; + + let index_name = "chaos_s1"; + setup_test_data(&cluster, index_name, 500).await?; + + // Kill node-1 (of 3 nodes, RF=2) + cluster.kill_meili(1).await?; + + let client = miroir_client(cluster.miroir_port()); + + // Searches should still work (RF=2 means surviving replicas cover all shards) + let results: serde_json::Value = client + .index(index_name) + .search() + .with_query("content") + .with_limit(500) + .execute() + .await?; + + let hits = results["hits"].as_array().unwrap(); + assert_eq!( + hits.len(), + 500, + "Search should return all 500 results after node loss" + ); + + // Check for degraded header (should not appear with RF=2 and one node down) + let http_client = reqwest::Client::new(); + let search_url = format!( + "http://localhost:{}/indexes/{}/search", + cluster.miroir_port(), + index_name + ); + let resp = http_client + .post(&search_url) + .header("Authorization", format!("Bearer {}", MASTER_KEY)) + .json(&json!({"q": "content", "limit": 500})) + .send() + .await?; + + // With RF=2, surviving replicas cover all shards, so no degraded header + assert!( + resp.headers().get("X-Miroir-Degraded").is_none(), + "X-Miroir-Degraded header should not appear with RF=2 and one node down" + ); + + // Writes should succeed + let task = client + .index(index_name) + .add_documents( + &[json!({"id": "new-doc", "title": "New", "content": "During failure"})], + None, + ) + .await?; + let _ = wait_for_task(&client, task).await?; + + cluster.down().await?; + Ok(()) +} + +// ============================================================================ +// Runbook: Scenario 2 - Kill 2 of 3 nodes (RF=2) +// Expected: Shard loss; 503 or partial per policy +// ============================================================================ + +/// Runbook: Kill 2 of 3 nodes (RF=2) +/// +/// **Expected Result:** Shard loss; 503 (Service Unavailable) or partial results per policy. +/// +/// **Manual Reproduction:** +/// ```bash +/// # Start the RF=2 cluster +/// docker-compose -f examples/docker-compose-dev-rf2.yml -p manual-s2 up -d +/// +/// # Kill node-1 and node-2 +/// docker stop manual-s2_meili-1_1 +/// docker stop manual-s2_meili-2_1 +/// +/// # Run searches - may return 503 or partial results +/// curl -i -X POST 'http://localhost:7710/indexes/test/search' \ +/// -H 'Authorization: Bearer dev-key' \ +/// -H 'Content-Type: application/json' \ +/// --data-binary '{"q": "content"}' +/// ``` +/// +/// **Expected Observables:** +/// - `miroir_router_search_errors_total{reason="unavailable_shard"}` - Increases +/// - `X-Miroir-Degraded` - MUST appear on successful responses +/// - HTTP status may be 503 or 200 with degraded results +/// +/// **Recovery:** Restart one node; searches return full results again. +#[tokio::test] +#[ignore = "Chaos test: requires docker-compose and takes 30+ seconds"] +async fn chaos_scenario_2_kill_two_nodes_rf2() -> Result<(), Box> { + let cluster = TestCluster::new("scenario2", 2); + cluster.up().await?; + + let index_name = "chaos_s2"; + setup_test_data(&cluster, index_name, 500).await?; + + // Kill node-1 and node-2 (of 3 nodes, RF=2) + cluster.kill_meili(1).await?; + cluster.kill_meili(2).await?; + + let client = miroir_client(cluster.miroir_port()); + let http_client = reqwest::Client::new(); + let search_url = format!( + "http://localhost:{}/indexes/{}/search", + cluster.miroir_port(), + index_name + ); + + // Search may fail with 503 or return partial results + let resp = http_client + .post(&search_url) + .header("Authorization", format!("Bearer {}", MASTER_KEY)) + .json(&json!({"q": "content"})) + .send() + .await?; + + // Either 503 (service unavailable) or partial results with degraded header + if resp.status() == StatusCode::SERVICE_UNAVAILABLE { + // Expected: some shards are unavailable + } else { + // Partial results - check for degraded header + assert!( + resp.headers().get("X-Miroir-Degraded").is_some(), + "X-Miroir-Degraded header must appear with 2 of 3 nodes down" + ); + + let results: serde_json::Value = resp.json().await?; + let hits = results["hits"].as_array().unwrap(); + // Should have partial results (less than full 500) + assert!( + hits.len() < 500, + "Should return partial results when 2 of 3 nodes are down" + ); + } + + cluster.down().await?; + Ok(()) +} + +// ============================================================================ +// Runbook: Scenario 3 - Kill 1 of 2 Miroir replicas +// Expected: Zero client-visible downtime +// ============================================================================ + +/// Runbook: Kill 1 of 2 Miroir replicas +/// +/// **Expected Result:** Zero client-visible downtime (if running multiple Miroir replicas +/// behind a load balancer). +/// +/// **Manual Reproduction:** +/// ```bash +/// # In a real deployment, Miroir runs as a Kubernetes Deployment/StatefulSet +/// kubectl delete pod miroir-0 -n miroir +/// +/// # Immediately run searches - should succeed (LB routes to surviving replica) +/// curl -X POST 'http://miroir-service.miroir.svc.cluster.local:7700/indexes/test/search' \ +/// -H 'Authorization: Bearer $MASTER_KEY' \ +/// -H 'Content-Type: application/json' \ +/// --data-binary '{"q": "test"}' +/// ``` +/// +/// **Expected Observables:** +/// - Zero search failures (if health check is properly configured) +/// - Brief latency spike during failover (< 1s typically) +/// - No `X-Miroir-Degraded` header (backend nodes are healthy) +/// +/// **Recovery:** Kubernetes automatically restarts the pod; no manual intervention needed. +#[tokio::test] +#[ignore = "Chaos test: requires docker-compose and takes 30+ seconds"] +async fn chaos_scenario_3_kill_miroir_replica() -> Result<(), Box> { + let cluster = TestCluster::new("scenario3", 1); + cluster.up().await?; + + let index_name = "chaos_s3"; + setup_test_data(&cluster, index_name, 500).await?; + + let client = miroir_client(cluster.miroir_port()); + + // Kill the Miroir orchestrator + cluster.kill_miroir().await?; + + // Give it a moment to fully stop + sleep(Duration::from_secs(1)).await; + + // Restart Miroir + cluster.restart_miroir().await?; + + // Searches should work immediately after recovery + let results: serde_json::Value = client + .index(index_name) + .search() + .with_query("content") + .with_limit(500) + .execute() + .await?; + + let hits = results["hits"].as_array().unwrap(); + assert_eq!( + hits.len(), + 500, + "Search should return all results after Miroir restart" + ); + + cluster.down().await?; + Ok(()) +} + +// ============================================================================ +// Runbook: Scenario 4 - tc netem delay 500ms on one node +// Expected: Searches slow by at most max shard latency; no errors +// ============================================================================ + +/// Runbook: Network delay (tc netem) on one node +/// +/// **Expected Result:** Searches slow by at most max shard latency; no errors. +/// With 500ms added delay, searches should complete in < 2 seconds total. +/// +/// **Manual Reproduction:** +/// ```bash +/// # Start the cluster +/// docker-compose -f examples/docker-compose-dev.yml -p manual-s4 up -d +/// +/// # Apply 500ms delay to meili-0 +/// docker exec manual-s4_meili-0_1 \ +/// tc qdisc add dev eth0 root netem delay 500ms +/// +/// # Run searches and measure latency - should succeed but be slower +/// time curl -X POST 'http://localhost:7700/indexes/test/search' \ +/// -H 'Authorization: Bearer dev-key' \ +/// -H 'Content-Type: application/json' \ +/// --data-binary '{"q": "content"}' +/// +/// # Remove delay +/// docker exec manual-s4_meili-0_1 tc qdisc del dev eth0 root +/// ``` +/// +/// **Expected Observables:** +/// - `miroir_router_search_latency_seconds_bucket` - Latency increases +/// - No search failures +/// - No timeout errors +/// +/// **Recovery:** Remove the netem qdisc; latency returns to baseline. +#[tokio::test] +#[ignore = "Chaos test: requires docker-compose and takes 30+ seconds"] +async fn chaos_scenario_4_netem_delay() -> Result<(), Box> { + let cluster = TestCluster::new("scenario4", 1); + cluster.up().await?; + + let index_name = "chaos_s4"; + setup_test_data(&cluster, index_name, 500).await?; + + // Apply 500ms delay to node-0 + cluster.apply_netem(0, 500).await?; + + let client = miroir_client(cluster.miroir_port()); + + // Measure search latency with delay + let start = std::time::Instant::now(); + let results: serde_json::Value = client + .index(index_name) + .search() + .with_query("content") + .with_limit(100) + .execute() + .await?; + let delayed_latency = start.elapsed(); + + let hits = results["hits"].as_array().unwrap(); + assert_eq!( + hits.len(), + 100, + "Search should return all results with netem delay" + ); + + // Clean up netem + cluster.remove_netem(0).await?; + + // Measure baseline latency + let start = std::time::Instant::now(); + let results: serde_json::Value = client + .index(index_name) + .search() + .with_query("content") + .with_limit(100) + .execute() + .await?; + let baseline_latency = start.elapsed(); + + let baseline_hits = results["hits"].as_array().unwrap(); + assert_eq!(baseline_hits.len(), 100); + + // Delayed search should be slower but still succeed + assert!( + delayed_latency > baseline_latency, + "Delayed search should be slower than baseline" + ); + + // But not excessively slower (max shard latency + some overhead) + assert!( + delayed_latency < Duration::from_secs(2), + "Delayed search should complete in < 2s, took {:?}", + delayed_latency + ); + + cluster.down().await?; + Ok(()) +} + +// ============================================================================ +// Runbook: Scenario 5 - Restart a killed node +// Expected: Miroir detects recovery within health check interval, resumes routing +// ============================================================================ + +/// Runbook: Restart a killed node +/// +/// **Expected Result:** Miroir detects recovery within health check interval (default 5s) +/// and resumes routing. No data loss; searches and writes work normally after recovery. +/// +/// **Manual Reproduction:** +/// ```bash +/// # Start the RF=2 cluster +/// docker-compose -f examples/docker-compose-dev-rf2.yml -p manual-s5 up -d +/// +/// # Kill node-1 +/// docker stop manual-s5_meili-1_1 +/// +/// # Verify searches still work (RF=2 provides coverage) +/// curl -X POST 'http://localhost:7710/indexes/test/search' \ +/// -H 'Authorization: Bearer dev-key' \ +/// -H 'Content-Type: application/json' \ +/// --data-binary '{"q": "content", "limit": 500}' +/// +/// # Restart node-1 +/// docker start manual-s5_meili-1_1 +/// +/// # Wait for health check to detect recovery (default: 5s interval) +/// sleep 10 +/// +/// # Verify searches work with full node set +/// curl -X POST 'http://localhost:7710/indexes/test/search' \ +/// -H 'Authorization: Bearer dev-key' \ +/// -H 'Content-Type: application/json' \ +/// --data-binary '{"q": "content", "limit": 500}' +/// ``` +/// +/// **Expected Observables:** +/// - `miroir_node_health_status{node="meili-1"}` - Goes to 0, then back to 1 +/// - `miroir_node_requests_total{node="meili-1"}` - Drops to 0, then increases +/// - No search failures during outage +/// +/// **Recovery:** Restart the failed node; Miroir resumes routing within health check interval. +#[tokio::test] +#[ignore = "Chaos test: requires docker-compose and takes 30+ seconds"] +async fn chaos_scenario_5_restart_node() -> Result<(), Box> { + let cluster = TestCluster::new("scenario5", 2); + cluster.up().await?; + + let index_name = "chaos_s5"; + setup_test_data(&cluster, index_name, 500).await?; + + let client = miroir_client(cluster.miroir_port()); + + // Kill node-1 + cluster.kill_meili(1).await?; + + // Searches still work with RF=2 + let results: serde_json::Value = client + .index(index_name) + .search() + .with_query("content") + .with_limit(500) + .execute() + .await?; + + let hits = results["hits"].as_array().unwrap(); + assert_eq!(hits.len(), 500); + + // Restart node-1 + cluster.restart_meili(1).await?; + + // Wait a bit for health check to detect recovery + sleep(Duration::from_secs(5)).await; + + // Searches should still work + let results: serde_json::Value = client + .index(index_name) + .search() + .with_query("content") + .with_limit(500) + .execute() + .await?; + + let hits = results["hits"].as_array().unwrap(); + assert_eq!( + hits.len(), + 500, + "Search should return all results after node recovery" + ); + + // Verify node is routing again by adding a new document + let task = client + .index(index_name) + .add_documents( + &[json!({"id": "after-recovery", "title": "After", "content": "Recovery test"})], + None, + ) + .await?; + let _ = wait_for_task(&client, task).await?; + + // Search for the new document + let results: serde_json::Value = client + .index(index_name) + .search() + .with_query("Recovery test") + .execute() + .await?; + + let hits = results["hits"].as_array().unwrap(); + assert_eq!( + hits.len(), + 1, + "Should find document added after node recovery" + ); + + cluster.down().await?; + Ok(()) +} + +// ============================================================================ +// Runbook: Scenario 6 - Kill a node mid-rebalance +// Expected: Rebalancer pauses, resumes on recovery; no data loss +// ============================================================================ + +/// Runbook: Kill a node mid-rebalance +/// +/// **Expected Result:** Rebalancer pauses, resumes on recovery; no data loss. +/// Write operation may fail or succeed partially. +/// +/// **Manual Reproduction:** +/// ```bash +/// # Start the RF=2 cluster +/// docker-compose -f examples/docker-compose-dev-rf2.yml -p manual-s6 up -d +/// +/// # Create a test index +/// curl -X POST 'http://localhost:7710/indexes' \ +/// -H 'Authorization: Bearer dev-key' \ +/// -H 'Content-Type: application/json' \ +/// --data-binary '{"uid": "manual-s6", "primaryKey": "id"}' +/// +/// # Start a large document load +/// # (while loading, kill node-1) +/// docker stop manual-s6_meili-1_1 +/// +/// # Restart node-1 +/// docker start manual-s6_meili-1_1 +/// +/// # Check document count +/// curl -X POST 'http://localhost:7710/indexes/manual-s6/search' \ +/// -H 'Authorization: Bearer dev-key' \ +/// -H 'Content-Type: application/json' \ +/// --data-binary '{"q": "", "limit": 1000}' +/// ``` +/// +/// **Expected Observables:** +/// - `miroir_rebalancer_active_migrations` - Non-zero before failure, may pause +/// - `miroir_rebalancer_paused_total` - Increases when node fails +/// - Task may show `succeeded`, `failed`, or `processing` +/// +/// **Recovery:** Restart the failed node; rebalancer resumes from checkpoint. +#[tokio::test] +#[ignore = "Chaos test: requires docker-compose and takes 30+ seconds"] +async fn chaos_scenario_6_kill_mid_rebalance() -> Result<(), Box> { + let cluster = TestCluster::new("scenario6", 2); + cluster.up().await?; + + let client = miroir_client(cluster.miroir_port()); + let index_name = "chaos_s6"; + + // Create index + let indexes = client.clone(); + match indexes.get_index(index_name).await { + Ok(_) => {} + Err(_) => { + let task = indexes.create_index(index_name, Some("id")).await?; + wait_for_task(&indexes, task).await?; + } + } + + // Start adding documents (this will be our "rebalance" simulation) + let mut docs = Vec::new(); + for i in 0..1000 { + docs.push(json!({ + "id": format!("doc-{:05}", i), + "title": format!("Document {}", i), + "content": format!("Content for document {}", i), + })); + } + + // Add documents - this will be our "rebalance" simulation + let index = client.index(index_name); + let task = index.add_documents(&docs, None).await?; + + // Kill node-1 mid-operation + cluster.kill_meili(1).await?; + + // Wait for task to complete (should succeed or fail gracefully) + let _ = wait_for_task(&client, task).await; + + // Restart node-1 + cluster.restart_meili(1).await?; + + // Wait for recovery + sleep(Duration::from_secs(5)).await; + + // Verify data integrity - all documents should be present + let results: serde_json::Value = client + .index(index_name) + .search() + .with_query("content") + .with_limit(1000) + .execute() + .await?; + + let hits = results["hits"].as_array().unwrap(); + + // Should have all or most documents (some may be lost if task failed) + assert!( + hits.len() >= 900, + "Should have most documents after mid-rebalance failure, got {}", + hits.len() + ); + + cluster.down().await?; + Ok(()) +} diff --git a/examples/docker-compose-dev-rf2.yml b/examples/docker-compose-dev-rf2.yml index 7cb692a..8ef5067 100644 --- a/examples/docker-compose-dev-rf2.yml +++ b/examples/docker-compose-dev-rf2.yml @@ -7,6 +7,8 @@ services: meili-0: image: getmeili/meilisearch:v1.37.0 container_name: miroir-meili-0 + cap_add: + - NET_ADMIN environment: - MEILI_ENV=development - MEILI_MASTER_KEY=dev-node-key @@ -25,6 +27,8 @@ services: meili-1: image: getmeili/meilisearch:v1.37.0 container_name: miroir-meili-1 + cap_add: + - NET_ADMIN environment: - MEILI_ENV=development - MEILI_MASTER_KEY=dev-node-key @@ -43,6 +47,8 @@ services: meili-2: image: getmeili/meilisearch:v1.37.0 container_name: miroir-meili-2 + cap_add: + - NET_ADMIN environment: - MEILI_ENV=development - MEILI_MASTER_KEY=dev-node-key @@ -61,6 +67,8 @@ services: meili-3: image: getmeili/meilisearch:v1.37.0 container_name: miroir-meili-3 + cap_add: + - NET_ADMIN environment: - MEILI_ENV=development - MEILI_MASTER_KEY=dev-node-key @@ -79,6 +87,8 @@ services: meili-4: image: getmeili/meilisearch:v1.37.0 container_name: miroir-meili-4 + cap_add: + - NET_ADMIN environment: - MEILI_ENV=development - MEILI_MASTER_KEY=dev-node-key @@ -97,6 +107,8 @@ services: meili-5: image: getmeili/meilisearch:v1.37.0 container_name: miroir-meili-5 + cap_add: + - NET_ADMIN environment: - MEILI_ENV=development - MEILI_MASTER_KEY=dev-node-key diff --git a/examples/docker-compose-dev.yml b/examples/docker-compose-dev.yml index 4efd475..e07638d 100644 --- a/examples/docker-compose-dev.yml +++ b/examples/docker-compose-dev.yml @@ -6,6 +6,8 @@ services: meili-0: image: getmeili/meilisearch:v1.37.0 container_name: miroir-meili-0 + cap_add: + - NET_ADMIN environment: - MEILI_ENV=development - MEILI_MASTER_KEY=dev-node-key @@ -24,6 +26,8 @@ services: meili-1: image: getmeili/meilisearch:v1.37.0 container_name: miroir-meili-1 + cap_add: + - NET_ADMIN environment: - MEILI_ENV=development - MEILI_MASTER_KEY=dev-node-key @@ -42,6 +46,8 @@ services: meili-2: image: getmeili/meilisearch:v1.37.0 container_name: miroir-meili-2 + cap_add: + - NET_ADMIN environment: - MEILI_ENV=development - MEILI_MASTER_KEY=dev-node-key diff --git a/tests/chaos/runbooks/scenario1.md b/tests/chaos/runbooks/scenario1.md new file mode 100644 index 0000000..4a76eef --- /dev/null +++ b/tests/chaos/runbooks/scenario1.md @@ -0,0 +1,101 @@ +# Runbook: Kill 1 of 3 nodes (RF=2) + +**Scenario ID:** chaos_scenario_1_kill_one_node_rf2 + +## Expected Result + +Continuous search; degraded writes warn via header (though with RF=2 and one node down, surviving replicas cover all shards, so degraded header may not appear). + +## Precondition Check + +- 3-node cluster with RF=2 +- All nodes healthy +- Test index with 500 documents indexed +- All documents searchable + +## Manual Reproduction Steps + +```bash +# Start the RF=2 cluster +cd /path/to/miroir +docker-compose -f examples/docker-compose-dev-rf2.yml -p miroir-manual-s1 up -d + +# Wait for Miroir to be healthy +curl http://localhost:7710/health + +# Create test index and add documents +curl -X POST 'http://localhost:7710/indexes' \ + -H 'Authorization: Bearer dev-key' \ + -H 'Content-Type: application/json' \ + --data-binary '{ + "uid": "manual-s1", + "primaryKey": "id" + }' + +# Add 500 documents (use a script or the miroir-ctl) + +# Kill node-1 +docker stop miroir-manual-s1_meili-1_1 + +# Run searches +curl -X POST 'http://localhost:7710/indexes/manual-s1/search' \ + -H 'Authorization: Bearer dev-key' \ + -H 'Content-Type: application/json' \ + --data-binary '{"q": "content", "limit": 500}' + +# Check for degraded header +curl -I -X POST 'http://localhost:7710/indexes/manual-s1/search' \ + -H 'Authorization: Bearer dev-key' \ + -H 'Content-Type: application/json' \ + --data-binary '{"q": "content"}' + +# Cleanup +docker-compose -f examples/docker-compose-dev-rf2.yml -p miroir-manual-s1 down -v +``` + +## Expected Observables + +### Metrics + +- `miroir_router_search_latency_*` - May increase slightly as requests retry +- `miroir_node_requests_total{node="meili-1"}` - Drops to zero (node is down) +- `miroir_node_requests_total{node="meili-0"}` and `{node="meili-2"}` - Increase (take over load) + +### Headers + +- `X-Miroir-Degraded` - Should NOT appear (RF=2 provides full coverage with one node down) + +### Client Errors + +- No search failures +- All 500 documents returned +- Write operations succeed + +## Recovery Procedure + +```bash +# Restart the killed node +docker start miroir-manual-s1_meili-1_1 + +# Wait for health check to detect recovery (default: 5s interval) +# Miroir will automatically resume routing to the recovered node + +# Verify routing is restored +curl http://localhost:7710/health +``` + +## How This Differs on HA (2+ Miroir replicas) + +With multiple Miroir replicas: + +- Client requests are load-balanced across replicas +- If one Miroir replica fails, others continue serving +- No client-visible downtime +- Failed replica restarts and rejoins the cluster automatically + +## Notes + +- RF=2 means each shard exists on 2 nodes +- Losing 1 of 3 nodes means all shards still have at least 1 replica +- This is the "happy path" failure scenario - minimal impact +- Monitor `miroir_node_health_status` for real-time node state diff --git a/tests/chaos/runbooks/scenario2.md b/tests/chaos/runbooks/scenario2.md new file mode 100644 index 0000000..3167799 --- /dev/null +++ b/tests/chaos/runbooks/scenario2.md @@ -0,0 +1,101 @@ +# Runbook: Kill 2 of 3 nodes (RF=2) + +**Scenario ID:** chaos_scenario_2_kill_two_nodes_rf2 + +## Expected Result + +Shard loss; 503 (Service Unavailable) or partial results per policy. + +## Precondition Check + +- 3-node cluster with RF=2 +- All nodes healthy +- Test index with 500 documents indexed +- All documents searchable + +## Manual Reproduction Steps + +```bash +# Start the RF=2 cluster +cd /path/to/miroir +docker-compose -f examples/docker-compose-dev-rf2.yml -p miroir-manual-s2 up -d + +# Wait for Miroir to be healthy +curl http://localhost:7710/health + +# Create test index and add documents (use miroir-ctl or script) + +# Kill node-1 and node-2 +docker stop miroir-manual-s2_meili-1_1 +docker stop miroir-manual-s2_meili-2_1 + +# Run searches +curl -X POST 'http://localhost:7710/indexes/manual-s2/search' \ + -H 'Authorization: Bearer dev-key' \ + -H 'Content-Type: application/json' \ + --data-binary '{"q": "content"}' + +# Check response status and headers +curl -i -X POST 'http://localhost:7710/indexes/manual-s2/search' \ + -H 'Authorization: Bearer dev-key' \ + -H 'Content-Type: application/json' \ + --data-binary '{"q": "content"}' + +# Cleanup +docker-compose -f examples/docker-compose-dev-rf2.yml -p miroir-manual-s2 down -v +``` + +## Expected Observables + +### Metrics + +- `miroir_router_search_errors_total{reason="unavailable_shard"}` - Increases +- `miroir_router_search_degraded_total` - Increases +- `miroir_node_requests_total{node="meili-0"}` - Spikes (sole survivor) +- `miroir_node_requests_total{node="meili-1"}` and `{node="meili-2"}` - Zero (down) + +### Headers + +- `X-Miroir-Degraded` - MUST appear (indicates partial results) +- HTTP status may be 503 or 200 with degraded results + +### Client Errors + +- Some searches may fail with 503 +- Successful searches return partial results (< 500 documents) +- Degraded header always present on successful partial results + +## Recovery Procedure + +```bash +# Restart one killed node (node-1) +docker start miroir-manual-s2_meili-1_1 + +# Wait for health check to detect recovery (default: 5s interval) +# Verify cluster is recovering +curl http://localhost:7710/health + +# Searches should now return full results +curl -X POST 'http://localhost:7710/indexes/manual-s2/search' \ + -H 'Authorization: Bearer dev-key' \ + -H 'Content-Type: application/json' \ + --data-binary '{"q": "content", "limit": 500}' +``` + +## How This Differs on HA (2+ Miroir replicas) + +With multiple Miroir replicas: + +- Same backend behavior (partial results or 503) +- Client may be routed to different Miroir instances +- Load balancer health checks prevent routing to failed Miroir replicas +- No additional backend redundancy - this is about Miroir itself, not Meilisearch nodes + +## Notes + +- RF=2 with 2 nodes down means many shards lose both replicas +- Remaining node (meili-0) can only serve documents it holds +- This is the "graceful degradation" scenario - partial results better than none +- Per plan §1 principle 5: degrade rather than fail entirely +- Alert on `miroir_router_search_degraded_total` increasing +- Consider RF=3 for deployments requiring tolerance of 2-node failures diff --git a/tests/chaos/runbooks/scenario3.md b/tests/chaos/runbooks/scenario3.md new file mode 100644 index 0000000..e148eef --- /dev/null +++ b/tests/chaos/runbooks/scenario3.md @@ -0,0 +1,108 @@ +# Runbook: Kill 1 of 2 Miroir replicas + +**Scenario ID:** chaos_scenario_3_kill_miroir_replica + +## Expected Result + +Zero client-visible downtime (if running multiple Miroir replicas behind a load balancer). + +## Precondition Check + +- Multiple Miroir replicas running (e.g., 2 replicas) +- Load balancer (Kubernetes Service, Traefik, nginx) configured +- Health check endpoint responding on all replicas +- Test index with documents indexed + +## Manual Reproduction Steps + +```bash +# In a real deployment, Miroir runs as a Kubernetes Deployment/StatefulSet +# This simulates killing a pod + +# Check current replicas +kubectl get pods -n miroir + +# Kill one Miroir pod (simulate crash) +kubectl delete pod miroir-0 -n miroir + +# Immediately run searches - should succeed +# Kubernetes Service automatically routes to surviving replica +curl -X POST 'http://miroir-service.miroir.svc.cluster.local:7700/indexes/test/search' \ + -H 'Authorization: Bearer $MASTER_KEY' \ + -H 'Content-Type: application/json' \ + --data-binary '{"q": "test"}' + +# Watch pod replacement +kubectl get pods -n miroir -w + +# After new pod is ready, verify it's receiving traffic +kubectl port-forward miroir-1 -n miroir 7700:7700 +curl http://localhost:7700/health +``` + +## Expected Observables + +### Metrics + +- `miroir_up` - Drops to 0 for killed replica, recovers when pod replaced +- `kiowala_request_duration_seconds` - May spike briefly during failover +- `kiowala_requests_total{pod="miroir-0"}` - Drops to zero +- `kiowala_requests_total{pod="miroir-1"}` - Increases (takes 100% of load) + +### Kubernetes + +- Pod `miroir-0` shows `Terminating` then disappears +- New pod `miroir-2` (or reusing `miroir-0`) appears and goes `Pending` → `Running` → `Ready` +- Service endpoints list updates automatically + +### Client Errors + +- Zero search failures (if health check is properly configured) +- Brief latency spike during failover (< 1s typically) +- No `X-Miroir-Degraded` header (backend nodes are healthy) + +## Recovery Procedure + +```bash +# Kubernetes automatically restarts the pod +# Verify the new pod is healthy +kubectl get pods -n miroir +kubectl describe pod miroir-2 -n miroir + +# Check logs if pod fails to start +kubectl logs miroir-2 -n miroir + +# Verify service endpoints +kubectl get endpoints miroir-service -n miroir +``` + +## How This Differs on Single Miroir Instance + +With only one Miroir instance: + +- Killing the instance causes total outage +- No failover until instance restarts +- Clients see connection errors +- This is why HA mode (2+ replicas) is recommended for production + +## Notes + +- Miroir itself is stateless in the request path +- All state is in the backend Meilisearch nodes +- Multiple Miroir replicas share the same backend cluster +- Health check at `/health` is critical for load balancer failover +- Consider using readiness/liveness probes in Kubernetes: + ```yaml + livenessProbe: + httpGet: + path: /health + port: 7700 + initialDelaySeconds: 10 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /health + port: 7700 + initialDelaySeconds: 5 + periodSeconds: 2 + ``` diff --git a/tests/chaos/runbooks/scenario4.md b/tests/chaos/runbooks/scenario4.md new file mode 100644 index 0000000..5de2643 --- /dev/null +++ b/tests/chaos/runbooks/scenario4.md @@ -0,0 +1,99 @@ +# Runbook: Network delay (tc netem) on one node + +**Scenario ID:** chaos_scenario_4_netem_delay + +## Expected Result + +Searches slow by at most max shard latency; no errors. With 500ms added delay, searches should complete in < 2 seconds total. + +## Precondition Check + +- 3-node cluster running +- All nodes healthy +- Test index with 500 documents indexed +- Docker containers have `CAP_NET_ADMIN` capability (required for tc netem) + +## Manual Reproduction Steps + +```bash +# Start the cluster +cd /path/to/miroir +docker-compose -f examples/docker-compose-dev.yml -p miroir-manual-s4 up -d + +# Wait for cluster to be healthy +curl http://localhost:7700/health + +# Apply 500ms delay to meili-0 +docker exec miroir-manual-s4_meili-0_1 \ + tc qdisc add dev eth0 root netem delay 500ms + +# Run searches and measure latency +time curl -X POST 'http://localhost:7700/indexes/test/search' \ + -H 'Authorization: Bearer dev-key' \ + -H 'Content-Type: application/json' \ + --data-binary '{"q": "content"}' + +# Remove delay +docker exec miroir-manual-s4_meili-0_1 \ + tc qdisc del dev eth0 root + +# Verify latency returns to baseline +time curl -X POST 'http://localhost:7700/indexes/test/search' \ + -H 'Authorization: Bearer dev-key' \ + -H 'Content-Type: application/json' \ + --data-binary '{"q": "content"}' + +# Cleanup +docker-compose -f examples/docker-compose-dev.yml -p miroir-manual-s4 down -v +``` + +## Expected Observables + +### Metrics + +- `miroir_router_search_latency_seconds_bucket` - Latency increases +- `miroir_node_request_duration_seconds{node="meili-0"}` - Increases by ~500ms +- `miroir_router_search_timeout_total` - Should NOT increase +- `miroir_router_search_errors_total` - Should NOT increase + +### Client Errors + +- No search failures +- All searches complete successfully +- Latency increases but remains within timeout threshold + +## Recovery Procedure + +```bash +# Remove the network delay +docker exec tc qdisc del dev eth0 root + +# Verify latency returns to baseline +# Monitor metrics for recovery +curl http://localhost:9090/api/v1/query?query=miroir_router_search_latency_seconds +``` + +## How This Differs on HA (2+ Miroir replicas) + +With multiple Miroir replicas: + +- Same behavior (backend node delay affects all replicas) +- No additional benefit from multiple Miroir instances +- Consider replica-aware routing if network partitions are common +- In cross-region deployments, use replica groups to route to nearest region + +## Notes + +- `tc netem` simulates network latency, packet loss, duplication, and more +- This test validates timeout configuration is appropriate +- Per plan §8: searches should complete in < 2× baseline latency +- Common causes of real network delay: + - Network congestion + - Cross-region traffic + - Oversubscribed links + - DNS delays +- If latency is consistently high, consider: + - Increasing timeout values + - Adding replica groups in same region as clients + - Using CDN or edge caching + - Investigating network path (traceroute, mtr) diff --git a/tests/chaos/runbooks/scenario5.md b/tests/chaos/runbooks/scenario5.md new file mode 100644 index 0000000..f08cb6f --- /dev/null +++ b/tests/chaos/runbooks/scenario5.md @@ -0,0 +1,129 @@ +# Runbook: Restart a killed node + +**Scenario ID:** chaos_scenario_5_restart_node + +## Expected Result + +Miroir detects recovery within health check interval (default 5s) and resumes routing. No data loss; searches and writes work normally after recovery. + +## Precondition Check + +- 3-node cluster with RF=2 +- All nodes healthy +- Test index with 500 documents indexed +- Kill and restart one node + +## Manual Reproduction Steps + +```bash +# Start the RF=2 cluster +cd /path/to/miroir +docker-compose -f examples/docker-compose-dev-rf2.yml -p miroir-manual-s5 up -d + +# Wait for cluster to be healthy +curl http://localhost:7710/health + +# Create test index and add documents + +# Kill node-1 +docker stop miroir-manual-s5_meili-1_1 + +# Verify searches still work (RF=2 provides coverage) +curl -X POST 'http://localhost:7710/indexes/test/search' \ + -H 'Authorization: Bearer dev-key' \ + -H 'Content-Type: application/json' \ + --data-binary '{"q": "content", "limit": 500}' + +# Restart node-1 +docker start miroir-manual-s5_meili-1_1 + +# Wait for health check to detect recovery (default: 5s interval) +sleep 10 + +# Verify searches work with full node set +curl -X POST 'http://localhost:7710/indexes/test/search' \ + -H 'Authorization: Bearer dev-key' \ + -H 'Content-Type: application/json' \ + --data-binary '{"q": "content", "limit": 500}' + +# Add a new document to verify routing to recovered node +curl -X POST 'http://localhost:7710/indexes/test/documents' \ + -H 'Authorization: Bearer dev-key' \ + -H 'Content-Type: application/json' \ + --data-binary '[{"id": "after-recovery", "title": "After Recovery"}]' + +# Search for the new document +curl -X POST 'http://localhost:7710/indexes/test/search' \ + -H 'Authorization: Bearer dev-key' \ + -H 'Content-Type: application/json' \ + --data-binary '{"q": "After Recovery"}' + +# Cleanup +docker-compose -f examples/docker-compose-dev-rf2.yml -p miroir-manual-s5 down -v +``` + +## Expected Observables + +### Metrics + +- `miroir_node_health_status{node="meili-1"}` - Goes to 0 (down), then back to 1 (up) +- `miroir_node_requests_total{node="meili-1"}` - Drops to 0, then increases after recovery +- `miroir_router_search_errors_total` - Should NOT increase +- `miroir_anti_entropy_migrations_total` - May increase if anti-entropy runs + +### Node State Transition + +1. Node is `Active` (healthy) +2. Node is killed → state becomes `Failed` +3. Health check marks node as unhealthy +4. Router stops routing to failed node +5. Node is restarted +6. Health check detects recovery (within 5s) +7. Node state returns to `Active` +8. Router resumes routing to recovered node + +### Client Errors + +- No search failures during outage (RF=2 provides coverage) +- No search failures after recovery +- New documents can be added and searched + +## Recovery Procedure + +```bash +# Restart the failed node +docker start + +# Verify node is running +docker ps | grep meili-1 + +# Check Miroir health endpoint +curl http://localhost:7710/health + +# Verify node is receiving traffic +# Watch metrics for the node +curl http://localhost:9090/api/v1/query?query=miroir_node_requests_total{node=\"meili-1\"} +``` + +## How This Differs on HA (2+ Miroir replicas) + +With multiple Miroir replicas: + +- Same backend behavior (node recovery is independent) +- Each Miroir replica detects recovery independently +- Health check interval applies per replica +- No difference in recovery time + +## Notes + +- Health check interval is configurable via `health_check_interval_seconds` +- Default is 5 seconds; adjust for faster/slower recovery detection +- Anti-entropy may run after node recovery to ensure data consistency +- If node was down for extended period, consider: + - Running manual anti-entropy via `miroir-ctl anti-entropy start` + - Checking for missed writes via task log reconciliation + - Monitoring for any shard inconsistencies +- This scenario is common during: + - Rolling upgrades (restart nodes one at a time) + - Node maintenance (patching, config changes) + - Crash recovery (OOM killed, segfault) diff --git a/tests/chaos/runbooks/scenario6.md b/tests/chaos/runbooks/scenario6.md new file mode 100644 index 0000000..adabba5 --- /dev/null +++ b/tests/chaos/runbooks/scenario6.md @@ -0,0 +1,133 @@ +# Runbook: Kill a node mid-rebalance + +**Scenario ID:** chaos_scenario_6_kill_mid_rebalance + +## Expected Result + +Rebalancer pauses, resumes on recovery; no data loss. Write operation may fail or succeed partially. + +## Precondition Check + +- 3-node cluster with RF=2 +- All nodes healthy +- Active rebalance operation in progress (node addition, drain, or bulk document load) + +## Manual Reproduction Steps + +```bash +# Start the RF=2 cluster +cd /path/to/miroir +docker-compose -f examples/docker-compose-dev-rf2.yml -p miroir-manual-s6 up -d + +# Wait for cluster to be healthy +curl http://localhost:7710/health + +# Create a test index +curl -X POST 'http://localhost:7710/indexes' \ + -H 'Authorization: Bearer dev-key' \ + -H 'Content-Type: application/json' \ + --data-binary '{"uid": "manual-s6", "primaryKey": "id"}' + +# Start a large document load (simulates rebalance traffic) +# Use a script to add 1000+ documents + +# While load is in progress, kill node-1 +docker stop miroir-manual-s6_meili-1_1 + +# Check the task status +curl 'http://localhost:7710/tasks?uids=1' \ + -H 'Authorization: Bearer dev-key' + +# Restart node-1 +docker start miroir-manual-s6_meili-1_1 + +# Wait for recovery +sleep 10 + +# Check document count +curl -X POST 'http://localhost:7710/indexes/manual-s6/search' \ + -H 'Authorization: Bearer dev-key' \ + -H 'Content-Type: application/json' \ + --data-binary '{"q": "", "limit": 1000}' + +# Cleanup +docker-compose -f examples/docker-compose-dev-rf2.yml -p miroir-manual-s6 down -v +``` + +## Expected Observables + +### Metrics + +- `miroir_rebalancer_active_migrations` - Non-zero before failure, may pause during failure +- `miroir_rebalancer_paused_total` - Increases when node fails +- `miroir_node_health_status{node="meili-1"}` - Goes to 0 (down), then back to 1 +- `miroir_task_errors_total` - May increase if write task fails + +### Rebalancer Behavior + +1. Rebalance starts (e.g., bulk document load or node addition) +2. Node failure detected mid-operation +3. Rebalancer pauses migrations involving failed node +4. Healthy migrations continue +5. Node recovers +6. Rebalancer resumes from checkpoint +7. Migrations complete successfully + +### Task Status + +- Task may show `succeeded`, `failed`, or `processing` +- Failed tasks can be retried via client +- Documents written before failure are preserved + +## Recovery Procedure + +```bash +# Restart the failed node +docker start + +# Wait for node to be healthy +curl http://localhost:7710/health + +# Check rebalancer status +curl 'http://localhost:7710/_miroir/rebalancer/status' \ + -H 'Authorization: Bearer $ADMIN_API_KEY' + +# If rebalancer is paused, resume it manually +curl -X POST 'http://localhost:7710/_miroir/rebalancer/resume' \ + -H 'Authorization: Bearer $ADMIN_API_KEY' + +# Run anti-entropy to ensure consistency +curl -X POST 'http://localhost:7710/_miroir/anti-entropy/start' \ + -H 'Authorization: Bearer $ADMIN_API_KEY' + +# Monitor anti-entropy progress +curl 'http://localhost:7710/_miroir/anti-entropy/status' \ + -H 'Authorization: Bearer $ADMIN_API_KEY' +``` + +## How This Differs on HA (2+ Miroir replicas) + +With multiple Miroir replicas: + +- Rebalancer runs in leader-elected mode (only one replica is leader) +- If leader replica fails, another replica takes over +- Rebalancer state is persisted in task store (Redis or SQLite) +- Recovery resumes from last checkpoint +- No duplicate migrations (leader election prevents split-brain) + +## Notes + +- Rebalancer is designed to tolerate failures gracefully +- Each migration has a checkpoint; can be resumed after failure +- Task store (Redis/SQLite) provides durability across restarts +- Leader election ensures only one rebalancer is active at a time +- Anti-entropy can verify and repair any inconsistencies after recovery +- This scenario validates: + - Rebalancer pause/resume logic + - Task store durability + - Leader election correctness + - No data loss or corruption +- Common during: + - Node failures during scale-out/scale-in + - Network partitions during rebalance + - Pod evictions during bulk operations