//! Integration tests for Miroir with docker-compose. //! //! These tests run against a real Miroir + Meilisearch stack started via docker-compose. //! Tests cover: //! - Document round-trip (1000 docs) //! - Search coverage across all shards (unique-keyword test) //! - Facet aggregation //! - Offset/limit paging //! - Settings broadcast //! - Task polling //! - Node failure with RF=2 //! //! Run with: //! cargo nextest run -E 'test(docker_compose_integration)' //! //! Prerequisites: //! Option 1: docker compose -f examples/docker-compose-dev.yml up -d //! Option 2: Set MIROIR_TEST_MIROIR_URL to point to a running Miroir instance //! Option 3: Set MIROIR_TEST_SKIP_DOCKER=1 to skip these tests //! //! Environment variables: //! - `MIROIR_TEST_MIROIR_URL`: If set, use this URL instead of localhost:7700 //! - `MIROIR_TEST_SKIP_DOCKER`: If set, skip tests that require Docker/Miroir //! //! See docs/TESTING.md for more details. use serde_json::json; use std::collections::HashSet; use std::time::Duration; /// Default base URL for Miroir API (from docker-compose) const DEFAULT_MIROIR_BASE_URL: &str = "http://localhost:7700"; /// Master key for authentication (from dev-config.yaml) const MASTER_KEY: &str = "dev-key"; /// Get the Miroir base URL from environment or default. /// /// Environment variables: /// - `MIROIR_TEST_MIROIR_URL`: If set, use this URL instead of the default /// - `MIROIR_TEST_SKIP_DOCKER`: If set, return an error (test should skip) /// /// Returns Ok(url) if Miroir is reachable, Err(skip_reason) if not. fn get_miroir_base_url() -> Result { // Check if Docker tests are explicitly skipped if std::env::var("MIROIR_TEST_SKIP_DOCKER").is_ok() { return Err("Docker tests skipped via MIROIR_TEST_SKIP_DOCKER. \ Set MIROIR_TEST_MIROIR_URL=http://your-miroir:7700 to test against external Miroir, \ or unset MIROIR_TEST_SKIP_DOCKER and ensure docker-compose is running." .to_string()); } // Use external URL if provided let url = if let Ok(url) = std::env::var("MIROIR_TEST_MIROIR_URL") { url } else { DEFAULT_MIROIR_BASE_URL.to_string() }; // Verify Miroir is actually reachable using a simple TCP check use std::net::ToSocketAddrs; use std::time::Duration; // Extract host and port from URL (assuming http://host:port format) let url_without_scheme = url .strip_prefix("http://") .or_else(|| url.strip_prefix("https://")) .unwrap_or(&url); let addrs: Vec = url_without_scheme .to_socket_addrs() .map_err(|e| format!("Failed to resolve address {url_without_scheme}: {e}"))? .collect(); if addrs.is_empty() { return Err(format!( "No addresses found for {url_without_scheme}. \ Ensure docker-compose stack is running: docker compose -f examples/docker-compose-dev.yml up -d. \ Or set MIROIR_TEST_MIROIR_URL to point to a running Miroir instance." )); } // Try each resolved address for socket_addr in addrs { if std::net::TcpStream::connect_timeout(&socket_addr, Duration::from_secs(2)).is_ok() { return Ok(url); } } Err(format!( "Failed to connect to Miroir at {url}. \ Ensure docker-compose stack is running: docker compose -f examples/docker-compose-dev.yml up -d. \ Or set MIROIR_TEST_MIROIR_URL to point to a running Miroir instance, \ or set MIROIR_TEST_SKIP_DOCKER=1 to skip." )) } /// Macro to skip test if Miroir/Docker is unavailable macro_rules! skip_if_no_miroir { () => { match get_miroir_base_url() { Ok(url) => url, Err(e) => { eprintln!("Skipping test: {e}"); return; } } }; } /// HTTP client for making requests #[derive(Clone)] struct HttpClient { client: reqwest::Client, base_url: String, master_key: String, } impl HttpClient { fn new(base_url: String) -> Self { let client = reqwest::Client::builder() .timeout(Duration::from_secs(30)) .connect_timeout(Duration::from_secs(5)) .build() .expect("Failed to create HTTP client"); Self { client, base_url, master_key: MASTER_KEY.to_string(), } } async fn get(&self, path: &str) -> Result<(u16, String), String> { let url = format!("{}{}", self.base_url, path); let response = self .client .get(&url) .header("Authorization", format!("Bearer {}", self.master_key)) .send() .await .map_err(|e| format!("GET {url} failed: {e}"))?; let status = response.status().as_u16(); let body = response .text() .await .map_err(|e| format!("Failed to read body: {e}"))?; Ok((status, body)) } async fn post(&self, path: &str, body: &serde_json::Value) -> Result<(u16, String), String> { let url = format!("{}{}", self.base_url, path); let response = self .client .post(&url) .header("Authorization", format!("Bearer {}", self.master_key)) .header("Content-Type", "application/json") .json(body) .send() .await .map_err(|e| format!("POST {url} failed: {e}"))?; let status = response.status().as_u16(); let body = response .text() .await .map_err(|e| format!("Failed to read body: {e}"))?; Ok((status, body)) } async fn patch(&self, path: &str, body: &serde_json::Value) -> Result<(u16, String), String> { let url = format!("{}{}", self.base_url, path); let response = self .client .patch(&url) .header("Authorization", format!("Bearer {}", self.master_key)) .header("Content-Type", "application/json") .json(body) .send() .await .map_err(|e| format!("PATCH {url} failed: {e}"))?; let status = response.status().as_u16(); let body = response .text() .await .map_err(|e| format!("Failed to read body: {e}"))?; Ok((status, body)) } async fn delete(&self, path: &str) -> Result<(u16, String), String> { let url = format!("{}{}", self.base_url, path); let response = self .client .delete(&url) .header("Authorization", format!("Bearer {}", self.master_key)) .send() .await .map_err(|e| format!("DELETE {url} failed: {e}"))?; let status = response.status().as_u16(); let body = response .text() .await .map_err(|e| format!("Failed to read body: {e}"))?; Ok((status, body)) } } /// Wait for a task to complete by polling the task endpoint async fn wait_for_task( client: &HttpClient, _index_uid: &str, task_uid: u64, timeout_secs: u64, ) -> Result { let start = std::time::Instant::now(); let timeout = Duration::from_secs(timeout_secs); while start.elapsed() < timeout { let (status, body) = client.get(&format!("/tasks/{task_uid}")).await?; if status == 200 { if let Ok(task) = serde_json::from_str::(&body) { if let Some(status_str) = task.get("status").and_then(|v| v.as_str()) { if status_str == "succeeded" { return Ok(task); } else if status_str == "failed" { return Err(format!("Task failed: {body}")); } } } } tokio::time::sleep(Duration::from_millis(100)).await; } Err(format!( "Task {task_uid} timed out after {timeout_secs} seconds" )) } /// Generate test documents with unique keywords for shard coverage testing fn generate_test_documents(count: usize) -> Vec { let colors = ["red", "green", "blue"]; (0..count) .map(|i| { json!({ "id": i, "title": format!("Document {}", i), "keyword": format!("keyword_{}", i % 16), // 16 unique keywords for 16 shards "color": colors[i % 3], // For facet testing "score": i % 100, "shard_hint": format!("shard_{}", i % 16), // Help verify shard distribution }) }) .collect() } /// Test 1: Document round-trip (1000 docs) #[tokio::test] async fn test_document_round_trip() { let base_url = skip_if_no_miroir!(); let client = HttpClient::new(base_url); // Create index let create_body = json!({ "uid": "round_trip_test", "primaryKey": "id" }); let (status, body) = client.post("/indexes", &create_body).await.unwrap(); assert!( status == 202 || status == 200, "Index creation failed: {body}" ); let task_uid: u64 = serde_json::from_str::(&body) .unwrap() .get("taskUid") .and_then(|v| v.as_u64()) .unwrap(); wait_for_task(&client, "round_trip_test", task_uid, 30) .await .unwrap(); // Index 1000 documents let docs = generate_test_documents(1000); let (status, body) = client .post("/indexes/round_trip_test/documents", &json!(docs)) .await .unwrap(); assert_eq!(status, 202, "Document indexing failed: {body}"); let task_uid: u64 = serde_json::from_str::(&body) .unwrap() .get("taskUid") .and_then(|v| v.as_u64()) .unwrap(); wait_for_task(&client, "round_trip_test", task_uid, 60) .await .unwrap(); // Verify all 1000 documents are retrievable by ID for i in 0..1000 { let (status, body) = client .get(&format!("/indexes/round_trip_test/documents/{i}")) .await .unwrap(); assert_eq!(status, 200, "Failed to fetch document {i}: {body}"); let doc: serde_json::Value = serde_json::from_str(&body).unwrap(); assert_eq!(doc.get("id").and_then(|v| v.as_i64()), Some(i64::from(i))); assert_eq!( doc.get("title").and_then(|v| v.as_str()), Some(format!("Document {i}").as_str()) ); } // Clean up let _ = client.delete("/indexes/round_trip_test").await; } /// Test 2: Search covers all shards (unique-keyword test) #[tokio::test] async fn test_search_shard_coverage() { let base_url = skip_if_no_miroir!(); let client = HttpClient::new(base_url); // Create index let create_body = json!({ "uid": "shard_coverage_test", "primaryKey": "id" }); let (status, body) = client.post("/indexes", &create_body).await.unwrap(); assert!( status == 202 || status == 200, "Index creation failed: {body}" ); let task_uid: u64 = serde_json::from_str::(&body) .unwrap() .get("taskUid") .and_then(|v| v.as_u64()) .unwrap(); wait_for_task(&client, "shard_coverage_test", task_uid, 30) .await .unwrap(); // Configure filterable attributes to enable shard filtering let settings_body = json!({ "filterableAttributes": ["keyword", "shard_hint", "color"] }); let (status, body) = client .patch("/indexes/shard_coverage_test/settings", &settings_body) .await .unwrap(); assert!( status == 202 || status == 200, "Settings update failed: {body}" ); let task_uid: u64 = serde_json::from_str::(&body) .unwrap() .get("taskUid") .and_then(|v| v.as_u64()) .unwrap(); wait_for_task(&client, "shard_coverage_test", task_uid, 30) .await .unwrap(); // Index documents let docs = generate_test_documents(1000); let (status, body) = client .post("/indexes/shard_coverage_test/documents", &json!(docs)) .await .unwrap(); assert_eq!(status, 202, "Document indexing failed: {body}"); let task_uid: u64 = serde_json::from_str::(&body) .unwrap() .get("taskUid") .and_then(|v| v.as_u64()) .unwrap(); wait_for_task(&client, "shard_coverage_test", task_uid, 60) .await .unwrap(); // Search for each unique keyword and verify we find all documents with that keyword let mut found_keywords: HashSet = HashSet::new(); for shard_id in 0..16 { let keyword = format!("keyword_{shard_id}"); let search_body = json!({ "q": keyword, "filter": format!("keyword = {}", keyword) }); let (status, body) = client .post("/indexes/shard_coverage_test/search", &search_body) .await .unwrap(); assert_eq!(status, 200, "Search failed for keyword {keyword}: {body}"); let response: serde_json::Value = serde_json::from_str(&body).unwrap(); let hits = response.get("hits").and_then(|v| v.as_array()).unwrap(); // Each keyword should appear in ~62-63 documents (1000 / 16) assert!( hits.len() >= 60 && hits.len() <= 65, "Expected ~62 documents for keyword {}, got {}", keyword, hits.len() ); found_keywords.insert(keyword); } // Verify we found all 16 keywords assert_eq!( found_keywords.len(), 16, "Expected to find all 16 keywords, found {}", found_keywords.len() ); // Clean up let _ = client.delete("/indexes/shard_coverage_test").await; } /// Test 3: Facet aggregation (3 colors, sum = 100) #[tokio::test] async fn test_facet_aggregation() { let base_url = skip_if_no_miroir!(); let client = HttpClient::new(base_url); // Create index let create_body = json!({ "uid": "facet_test", "primaryKey": "id" }); let (status, body) = client.post("/indexes", &create_body).await.unwrap(); assert!( status == 202 || status == 200, "Index creation failed: {body}" ); let task_uid: u64 = serde_json::from_str::(&body) .unwrap() .get("taskUid") .and_then(|v| v.as_u64()) .unwrap(); wait_for_task(&client, "facet_test", task_uid, 30) .await .unwrap(); // Configure filterable attributes let settings_body = json!({ "filterableAttributes": ["color"] }); let (status, body) = client .patch("/indexes/facet_test/settings", &settings_body) .await .unwrap(); assert!( status == 202 || status == 200, "Settings update failed: {body}" ); let task_uid: u64 = serde_json::from_str::(&body) .unwrap() .get("taskUid") .and_then(|v| v.as_u64()) .unwrap(); wait_for_task(&client, "facet_test", task_uid, 30) .await .unwrap(); // Index exactly 100 documents with known color distribution let colors = ["red", "green", "blue"]; let docs: Vec = (0..100) .map(|i| { json!({ "id": i, "color": colors[i % 3], }) }) .collect(); let (status, body) = client .post("/indexes/facet_test/documents", &json!(docs)) .await .unwrap(); assert_eq!(status, 202, "Document indexing failed: {body}"); let task_uid: u64 = serde_json::from_str::(&body) .unwrap() .get("taskUid") .and_then(|v| v.as_u64()) .unwrap(); wait_for_task(&client, "facet_test", task_uid, 30) .await .unwrap(); // Search with facet distribution let search_body = json!({ "q": "", "facets": ["color"] }); let (status, body) = client .post("/indexes/facet_test/search", &search_body) .await .unwrap(); assert_eq!(status, 200, "Facet search failed: {body}"); let response: serde_json::Value = serde_json::from_str(&body).unwrap(); let facet_distribution = response .get("facetDistribution") .and_then(|v| v.as_object()) .and_then(|o| o.get("color")) .and_then(|v| v.as_object()) .expect("No facet distribution found"); let red_count = facet_distribution .get("red") .and_then(|v| v.as_u64()) .unwrap_or(0); let green_count = facet_distribution .get("green") .and_then(|v| v.as_u64()) .unwrap_or(0); let blue_count = facet_distribution .get("blue") .and_then(|v| v.as_u64()) .unwrap_or(0); let total = red_count + green_count + blue_count; assert_eq!( total, 100, "Expected total of 100 documents across colors, got {total}" ); // Each color should have approximately equal distribution (33-34 each) assert!( (32..=35).contains(&red_count), "Expected ~33 red documents, got {red_count}" ); assert!( (32..=35).contains(&green_count), "Expected ~33 green documents, got {green_count}" ); assert!( (32..=35).contains(&blue_count), "Expected ~33 blue documents, got {blue_count}" ); // Clean up let _ = client.delete("/indexes/facet_test").await; } /// Test 4: Offset/limit paging #[tokio::test] async fn test_offset_limit_paging() { let base_url = skip_if_no_miroir!(); let client = HttpClient::new(base_url); // Create index let create_body = json!({ "uid": "paging_test", "primaryKey": "id" }); let (status, body) = client.post("/indexes", &create_body).await.unwrap(); assert!( status == 202 || status == 200, "Index creation failed: {body}" ); let task_uid: u64 = serde_json::from_str::(&body) .unwrap() .get("taskUid") .and_then(|v| v.as_u64()) .unwrap(); wait_for_task(&client, "paging_test", task_uid, 30) .await .unwrap(); // Index documents let docs: Vec = (0..100) .map(|i| { json!({ "id": i, "title": format!("Document {}", i), }) }) .collect(); let (status, body) = client .post("/indexes/paging_test/documents", &json!(docs)) .await .unwrap(); assert_eq!(status, 202, "Document indexing failed: {body}"); let task_uid: u64 = serde_json::from_str::(&body) .unwrap() .get("taskUid") .and_then(|v| v.as_u64()) .unwrap(); wait_for_task(&client, "paging_test", task_uid, 30) .await .unwrap(); // Test paging with offset and limit let page_size = 20; let mut all_ids: Vec = Vec::new(); for page in 0..5 { let offset = page * page_size; let search_body = json!({ "q": "", "limit": page_size, "offset": offset, "sort": ["id:asc"] }); let (status, body) = client .post("/indexes/paging_test/search", &search_body) .await .unwrap(); assert_eq!(status, 200, "Search failed for page {page}: {body}"); let response: serde_json::Value = serde_json::from_str(&body).unwrap(); let hits = response.get("hits").and_then(|v| v.as_array()).unwrap(); for hit in hits { if let Some(id) = hit.get("id").and_then(|v| v.as_i64()) { all_ids.push(id); } } // Last page might have fewer results let expected_count = if page < 4 { page_size } else { 0 }; if page < 4 { assert_eq!( hits.len(), expected_count, "Expected {} results on page {}, got {}", expected_count, page, hits.len() ); } } // Verify we got all 100 IDs and they're unique and sequential assert_eq!( all_ids.len(), 100, "Expected 100 total documents, got {}", all_ids.len() ); assert_eq!( all_ids, (0..100).collect::>(), "Documents are not in correct order" ); // Clean up let _ = client.delete("/indexes/paging_test").await; } /// Test 5: Settings broadcast #[tokio::test] async fn test_settings_broadcast() { let base_url = skip_if_no_miroir!(); let client = HttpClient::new(base_url); // Create index let create_body = json!({ "uid": "settings_test", "primaryKey": "id" }); let (status, body) = client.post("/indexes", &create_body).await.unwrap(); assert!( status == 202 || status == 200, "Index creation failed: {body}" ); let task_uid: u64 = serde_json::from_str::(&body) .unwrap() .get("taskUid") .and_then(|v| v.as_u64()) .unwrap(); wait_for_task(&client, "settings_test", task_uid, 30) .await .unwrap(); // Update settings let settings_body = json!({ "searchableAttributes": ["title", "description"], "filterableAttributes": ["color", "size"], "sortableAttributes": ["id", "score"], "rankingRules": ["words", "typo", "proximity", "attribute", "sort", "exactness"] }); let (status, body) = client .patch("/indexes/settings_test/settings", &settings_body) .await .unwrap(); assert!( status == 202 || status == 200, "Settings update failed: {body}" ); let task_uid: u64 = serde_json::from_str::(&body) .unwrap() .get("taskUid") .and_then(|v| v.as_u64()) .unwrap(); wait_for_task(&client, "settings_test", task_uid, 30) .await .unwrap(); // Verify settings were applied let (status, body) = client.get("/indexes/settings_test/settings").await.unwrap(); assert_eq!(status, 200, "Failed to get settings: {body}"); let settings: serde_json::Value = serde_json::from_str(&body).unwrap(); let searchable = settings .get("searchableAttributes") .and_then(|v| v.as_array()) .unwrap(); assert_eq!(searchable.len(), 2, "Expected 2 searchable attributes"); assert!(searchable.iter().any(|v| v.as_str() == Some("title"))); assert!(searchable.iter().any(|v| v.as_str() == Some("description"))); let filterable = settings .get("filterableAttributes") .and_then(|v| v.as_array()) .unwrap(); assert_eq!(filterable.len(), 2, "Expected 2 filterable attributes"); assert!(filterable.iter().any(|v| v.as_str() == Some("color"))); assert!(filterable.iter().any(|v| v.as_str() == Some("size"))); // Clean up let _ = client.delete("/indexes/settings_test").await; } /// Test 6: Task polling #[tokio::test] async fn test_task_polling() { let base_url = skip_if_no_miroir!(); let client = HttpClient::new(base_url); // Create index let create_body = json!({ "uid": "task_polling_test", "primaryKey": "id" }); let (status, body) = client.post("/indexes", &create_body).await.unwrap(); assert!( status == 202 || status == 200, "Index creation failed: {body}" ); let task_uid: u64 = serde_json::from_str::(&body) .unwrap() .get("taskUid") .and_then(|v| v.as_u64()) .unwrap(); // Poll task until completion let task = wait_for_task(&client, "task_polling_test", task_uid, 30) .await .unwrap(); // Verify task structure assert_eq!(task.get("uid").and_then(|v| v.as_u64()), Some(task_uid)); assert_eq!( task.get("status").and_then(|v| v.as_str()), Some("succeeded") ); assert_eq!( task.get("type").and_then(|v| v.as_str()), Some("indexCreation") ); // Index documents and poll let docs: Vec = (0..10) .map(|i| { json!({ "id": i, "title": format!("Document {}", i), }) }) .collect(); let (status, body) = client .post("/indexes/task_polling_test/documents", &json!(docs)) .await .unwrap(); assert_eq!(status, 202, "Document indexing failed: {body}"); let task_uid: u64 = serde_json::from_str::(&body) .unwrap() .get("taskUid") .and_then(|v| v.as_u64()) .unwrap(); let task = wait_for_task(&client, "task_polling_test", task_uid, 30) .await .unwrap(); assert_eq!( task.get("status").and_then(|v| v.as_str()), Some("succeeded") ); assert_eq!( task.get("type").and_then(|v| v.as_str()), Some("documentAdditionOrUpdate") ); // Clean up let _ = client.delete("/indexes/task_polling_test").await; } /// Test 7: Health check #[tokio::test] async fn test_health_check() { let base_url = skip_if_no_miroir!(); let client = HttpClient::new(base_url); let (status, body) = client.get("/health").await.unwrap(); assert_eq!(status, 200, "Health check failed: {body}"); let health: serde_json::Value = serde_json::from_str(&body).unwrap(); assert_eq!( health.get("status").and_then(|v| v.as_str()), Some("available") ); } /// Test 8: Direct Meilisearch node access (for debugging) #[tokio::test] async fn test_direct_meilisearch_access() { // This test accesses Meilisearch node 0 directly (port 7701 by default) // Skip if Docker tests are disabled let _base_url = skip_if_no_miroir!(); // For direct Meilisearch access, we use the default port 7701 // This is independent of the Miroir URL but we use the same skip logic let client = reqwest::Client::builder() .timeout(Duration::from_secs(5)) .build() .unwrap(); let response = client .get("http://localhost:7701/health") .header("Authorization", "Bearer dev-node-key") .send() .await .unwrap(); assert_eq!(response.status().as_u16(), 200); } /// Test 9: Node failure with RF=2 /// /// This test requires the RF=2 docker-compose stack: /// docker compose -f examples/docker-compose-dev-rf2.yml up -d /// /// Or set MIROIR_TEST_MIROIR_URL=http://localhost:7710 to test against RF=2 stack /// /// The test: /// 1. Indexes documents with RF=2 (each document replicated to both groups) /// 2. Stops a Meilisearch node mid-test (docker stop) /// 3. Verifies that searches still work using remaining replicas /// 4. Restarts the node and verifies recovery #[tokio::test] #[ignore] // Run with: cargo nextest run -E 'test(node_failure_rf2)' --ignored async fn test_node_failure_rf2() { // Use port 7710 for RF=2 stack, or allow override via environment let base_url = std::env::var("MIROIR_TEST_MIROIR_URL") .unwrap_or_else(|_| "http://localhost:7710".to_string()); // Check if Docker tests are skipped if std::env::var("MIROIR_TEST_SKIP_DOCKER").is_ok() { eprintln!("Skipping test: Docker tests skipped via MIROIR_TEST_SKIP_DOCKER"); return; } let client = HttpClient { client: reqwest::Client::builder() .timeout(Duration::from_secs(30)) .connect_timeout(Duration::from_secs(5)) .build() .expect("Failed to create HTTP client"), base_url, master_key: MASTER_KEY.to_string(), }; // Create index let create_body = json!({ "uid": "node_failure_test", "primaryKey": "id" }); let (status, body) = client.post("/indexes", &create_body).await.unwrap(); assert!( status == 202 || status == 200, "Index creation failed: {body}" ); let task_uid: u64 = serde_json::from_str::(&body) .unwrap() .get("taskUid") .and_then(|v| v.as_u64()) .unwrap(); wait_for_task(&client, "node_failure_test", task_uid, 30) .await .unwrap(); // Index 100 documents let docs: Vec = (0..100) .map(|i| { json!({ "id": i, "title": format!("Document {}", i), "group": i % 2, // Track which group this document belongs to }) }) .collect(); let (status, body) = client .post("/indexes/node_failure_test/documents", &json!(docs)) .await .unwrap(); assert_eq!(status, 202, "Document indexing failed: {body}"); let task_uid: u64 = serde_json::from_str::(&body) .unwrap() .get("taskUid") .and_then(|v| v.as_u64()) .unwrap(); wait_for_task(&client, "node_failure_test", task_uid, 60) .await .unwrap(); // Verify all documents are searchable let search_body = json!({ "q": "", "limit": 100 }); let (status, body) = client .post("/indexes/node_failure_test/search", &search_body) .await .unwrap(); assert_eq!(status, 200, "Search failed: {body}"); let response: serde_json::Value = serde_json::from_str(&body).unwrap(); let hits = response.get("hits").and_then(|v| v.as_array()).unwrap(); assert_eq!( hits.len(), 100, "Expected 100 documents before node failure" ); // Stop meili-1 (a node in replica group 0) // This simulates a node failure - RF=2 means we still have 1 replica in group 0 let output = std::process::Command::new("docker") .args(["stop", "miroir-meili-1"]) .output() .expect("Failed to stop container"); assert!( output.status.success(), "Failed to stop meili-1: {:?}", String::from_utf8_lossy(&output.stderr) ); // Wait for Miroir to detect the failure tokio::time::sleep(Duration::from_secs(5)).await; // Search should still work with degraded availability // Some documents may be missing if they were only on the failed node let (status, body) = client .post("/indexes/node_failure_test/search", &search_body) .await .unwrap(); assert_eq!(status, 200, "Search failed after node failure: {body}"); let response: serde_json::Value = serde_json::from_str(&body).unwrap(); let hits_after = response.get("hits").and_then(|v| v.as_array()).unwrap(); // With RF=2 and 1 node down, we should still get results // (each document has 2 replicas, 1 in each group) // Since we stopped a node in group 0, documents with replicas on that node // should still be accessible via the other replica in group 1 assert!( !hits_after.is_empty(), "Expected some results after node failure" ); // Restart the node let output = std::process::Command::new("docker") .args(["start", "miroir-meili-1"]) .output() .expect("Failed to start container"); assert!( output.status.success(), "Failed to start meili-1: {:?}", String::from_utf8_lossy(&output.stderr) ); // Wait for the node to recover and Miroir to detect it tokio::time::sleep(Duration::from_secs(10)).await; // After recovery, all documents should be accessible again let (status, body) = client .post("/indexes/node_failure_test/search", &search_body) .await .unwrap(); assert_eq!(status, 200, "Search failed after node recovery: {body}"); let response: serde_json::Value = serde_json::from_str(&body).unwrap(); let hits_recovered = response.get("hits").and_then(|v| v.as_array()).unwrap(); assert_eq!( hits_recovered.len(), 100, "Expected all 100 documents after node recovery" ); // Clean up let _ = client.delete("/indexes/node_failure_test").await; }