diff --git a/Cargo.lock b/Cargo.lock index a9e341c..36bb5dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,17 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "version_check", +] + [[package]] name = "ahash" version = "0.8.12" @@ -29,6 +40,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "1.0.0" @@ -79,6 +99,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "anyerror" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71add24cc141a1e8326f249b74c41cfd217aeb2a67c9c6cf9134d175469afd49" +dependencies = [ + "serde", +] + [[package]] name = "anyhow" version = "1.0.102" @@ -91,6 +120,12 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "async-trait" version = "0.1.89" @@ -99,7 +134,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -108,6 +143,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + [[package]] name = "axum" version = "0.7.9" @@ -175,6 +216,26 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bitflags" version = "2.11.1" @@ -184,6 +245,18 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -193,12 +266,70 @@ dependencies = [ "generic-array", ] +[[package]] +name = "borsh" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "bumpalo" version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "byte-unit" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c6d47a4e2961fb8721bcfc54feae6455f2f64e7054f9bc67e875f0e77f4c58d" +dependencies = [ + "rust_decimal", + "schemars", + "serde", + "utf8-width", +] + +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bytes" version = "1.11.1" @@ -227,6 +358,19 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + [[package]] name = "clap" version = "4.6.1" @@ -258,7 +402,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -321,6 +465,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -346,6 +496,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "unicode-xid", +] + [[package]] name = "diff" version = "0.1.13" @@ -391,7 +562,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -403,6 +574,12 @@ dependencies = [ "const-random", ] +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -461,6 +638,27 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -468,6 +666,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -476,6 +675,40 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + [[package]] name = "futures-task" version = "0.3.32" @@ -488,8 +721,13 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] @@ -544,13 +782,22 @@ dependencies = [ "wasip3", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] + [[package]] name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash", + "ahash 0.8.12", "allocator-api2", ] @@ -689,6 +936,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -921,6 +1192,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matchers" version = "0.2.0" @@ -969,8 +1246,12 @@ dependencies = [ name = "miroir-core" version = "0.1.0" dependencies = [ + "bincode", + "config", + "openraft", "serde", "serde_json", + "serde_yaml", "thiserror 2.0.18", "tracing", "twox-hash", @@ -1032,6 +1313,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1044,6 +1334,42 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "openraft" +version = "0.9.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2f40b0e23149ea8673591325bff055939f7a59f4f9a28476f0854282444ab3" +dependencies = [ + "anyerror", + "byte-unit", + "chrono", + "clap", + "derive_more", + "futures", + "maplit", + "openraft-macros", + "rand 0.8.6", + "serde", + "thiserror 1.0.69", + "tokio", + "tracing", + "tracing-futures", + "validit", +] + +[[package]] +name = "openraft-macros" +version = "0.9.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95b0033052d96bc535b99ea1f38973fad984ed658940bbd27ff59c377cada11c" +dependencies = [ + "chrono", + "proc-macro2", + "quote", + "semver", + "syn 2.0.117", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -1125,7 +1451,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1138,6 +1464,26 @@ dependencies = [ "sha2", ] +[[package]] +name = "pin-project" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1179,7 +1525,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit 0.25.11+spec-1.1.0", ] [[package]] @@ -1212,6 +1567,26 @@ version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "quinn" version = "0.11.9" @@ -1241,7 +1616,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -1288,14 +1663,41 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ - "rand_chacha", - "rand_core", + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", ] [[package]] @@ -1305,7 +1707,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", ] [[package]] @@ -1337,6 +1748,26 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "regex-automata" version = "0.4.14" @@ -1354,6 +1785,15 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + [[package]] name = "reqwest" version = "0.12.28" @@ -1406,6 +1846,35 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "ron" version = "0.8.1" @@ -1428,6 +1897,23 @@ dependencies = [ "ordered-multimap", ] +[[package]] +name = "rust_decimal" +version = "1.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ce901f9a19d251159075a4c37af514c3b8ef99c22e02dd8c19161cf397ee94a" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "rand 0.8.6", + "rkyv", + "serde", + "serde_json", + "wasm-bindgen", +] + [[package]] name = "rustc-hash" version = "2.1.2" @@ -1494,12 +1980,30 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "semver" version = "1.0.28" @@ -1533,7 +2037,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1581,6 +2085,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha2" version = "0.10.9" @@ -1617,6 +2134,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "slab" version = "0.4.12" @@ -1657,6 +2180,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.117" @@ -1685,9 +2219,15 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempfile" version = "3.27.0" @@ -1727,7 +2267,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1738,7 +2278,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1809,7 +2349,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1830,8 +2370,8 @@ checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", "serde_spanned", - "toml_datetime", - "toml_edit", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", ] [[package]] @@ -1843,6 +2383,15 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_edit" version = "0.22.27" @@ -1852,9 +2401,30 @@ dependencies = [ "indexmap", "serde", "serde_spanned", - "toml_datetime", + "toml_datetime 0.6.11", "toml_write", - "winnow", + "winnow 0.7.15", +] + +[[package]] +name = "toml_edit" +version = "0.25.11+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +dependencies = [ + "indexmap", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "winnow 1.0.1", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow 1.0.1", ] [[package]] @@ -1929,7 +2499,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1942,6 +2512,16 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + [[package]] name = "tracing-log" version = "0.2.0" @@ -1983,7 +2563,7 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" dependencies = [ - "rand", + "rand 0.9.4", ] [[package]] @@ -2016,12 +2596,24 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "url" version = "2.5.8" @@ -2034,6 +2626,12 @@ dependencies = [ "serde", ] +[[package]] +name = "utf8-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -2058,6 +2656,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "validit" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4efba0434d5a0a62d4f22070b44ce055dc18cb64d4fa98276aa523dadfaba0e7" +dependencies = [ + "anyerror", +] + [[package]] name = "valuable" version = "0.1.1" @@ -2070,6 +2677,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "want" version = "0.3.1" @@ -2145,7 +2758,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -2221,12 +2834,65 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -2384,6 +3050,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -2420,7 +3095,7 @@ dependencies = [ "heck", "indexmap", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -2436,7 +3111,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -2484,6 +3159,15 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "yaml-rust2" version = "0.8.1" @@ -2520,7 +3204,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -2541,7 +3225,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2561,7 +3245,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -2601,7 +3285,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/crates/miroir-core/benches/reshard_load.rs b/crates/miroir-core/benches/reshard_load.rs new file mode 100644 index 0000000..44f3dec --- /dev/null +++ b/crates/miroir-core/benches/reshard_load.rs @@ -0,0 +1,236 @@ +//! Resharding load benchmark: runs the plan §15 OP#3 test matrix. +//! +//! Validates the 2× transient storage/write load caveat under varied +//! doc sizes, corpus sizes, write rates, and topology configurations. + +use miroir_core::reshard::{simulate, SimParams}; + +const GB: u64 = 1024 * 1024 * 1024; +const TB: u64 = 1024 * GB; +const KB: u64 = 1024; +const MB: u64 = 1024 * KB; + +fn main() { + let test_matrix: Vec<(&str, SimParams)> = vec![ + ( + "Small docs, moderate corpus", + SimParams { + doc_size_bytes: KB, + corpus_size_bytes: 10 * GB, + write_rate_dps: 100, + replica_groups: 2, + replication_factor: 1, + old_shards: 64, + new_shards: 128, + nodes_per_group: 3, + backfill_throttle_dps: 10_000, + }, + ), + ( + "Medium docs, large corpus", + SimParams { + doc_size_bytes: 10 * KB, + corpus_size_bytes: 100 * GB, + write_rate_dps: 1000, + replica_groups: 2, + replication_factor: 2, + old_shards: 64, + new_shards: 128, + nodes_per_group: 4, + backfill_throttle_dps: 10_000, + }, + ), + ( + "Large blobs, very large corpus", + SimParams { + doc_size_bytes: MB, + corpus_size_bytes: TB, + write_rate_dps: 10, + replica_groups: 2, + replication_factor: 1, + old_shards: 64, + new_shards: 128, + nodes_per_group: 4, + backfill_throttle_dps: 5_000, + }, + ), + ]; + + println!("╔══════════════════════════════════════════════════════════════════╗"); + println!("║ Resharding Load Benchmark — Plan §15 OP#3 Validation ║"); + println!("╚══════════════════════════════════════════════════════════════════╝"); + println!(); + + let mut results = Vec::new(); + + for (label, params) in &test_matrix { + let start = std::time::Instant::now(); + let result = simulate(params); + let elapsed = start.elapsed(); + results.push((label, params.clone(), result, elapsed)); + } + + // Print summary table. + println!( + "{:<30} {:>10} {:>10} {:>8} {:>8} {:>14} {:>14} {:>10} {:>10}", + "Scenario", "DocSize", "Corpus", "RG", "RF", "StorageAmp", "PeakWriteAmp", "OldCV", "NewCV" + ); + println!("{}", "-".repeat(130)); + + for (label, params, result, elapsed) in &results { + let doc_size_str = if params.doc_size_bytes >= MB { + format!("{}MB", params.doc_size_bytes / MB) + } else { + format!("{}KB", params.doc_size_bytes / KB) + }; + let corpus_str = if params.corpus_size_bytes >= TB { + format!("{}TB", params.corpus_size_bytes / TB) + } else { + format!("{}GB", params.corpus_size_bytes / GB) + }; + + println!( + "{:<30} {:>10} {:>10} {:>8} {:>8} {:>14.2}× {:>14.2}× {:>10.4} {:>10.4}", + label, + doc_size_str, + corpus_str, + params.replica_groups, + params.replication_factor, + result.storage_amplification, + result.peak_write_amplification, + result.old_shard_cv, + result.new_shard_cv, + ); + println!(" (computed in {:.2?})", elapsed); + } + + println!(); + println!("═══════════════════════════════════════════════════════════════════"); + println!("Detailed Results (JSON)"); + println!("═══════════════════════════════════════════════════════════════════"); + println!(); + + for (label, params, result, _elapsed) in &results { + println!("--- {} ---", label); + println!(" doc_size: {} bytes", params.doc_size_bytes); + println!( + " corpus: {} bytes ({:.2} GB)", + params.corpus_size_bytes, + params.corpus_size_bytes as f64 / GB as f64 + ); + println!(" total_docs: {}", result.total_docs); + println!(" write_rate: {} dps", params.write_rate_dps); + println!( + " topology: RG={}, RF={}, nodes/group={}", + params.replica_groups, params.replication_factor, params.nodes_per_group + ); + println!( + " old_shards: {} → new_shards: {}", + params.old_shards, params.new_shards + ); + println!(); + println!(" STORAGE:"); + println!( + " normal (steady-state): {:.4} GB", + result.normal_storage_bytes as f64 / GB as f64 + ); + println!( + " peak (resharding): {:.4} GB", + result.peak_storage_bytes as f64 / GB as f64 + ); + println!( + " amplification: {:.2}×", + result.storage_amplification + ); + println!( + " per-node normal: {:.4} GB", + result.per_node_normal_storage_bytes as f64 / GB as f64 + ); + println!( + " per-node peak: {:.4} GB", + result.per_node_peak_storage_bytes as f64 / GB as f64 + ); + println!(); + println!(" WRITE LOAD:"); + println!( + " normal rate: {} writes/sec", + result.normal_write_rate + ); + println!( + " dual-write: {} writes/sec ({:.1}×)", + result.dual_write_rate, result.dual_write_amplification + ); + println!( + " peak (bf+dw): {} writes/sec ({:.2}×)", + result.peak_write_rate, result.peak_write_amplification + ); + println!(); + println!(" BACKFILL:"); + println!(" throttle: {} docs/sec", params.backfill_throttle_dps); + println!( + " duration: {:.2} seconds ({:.2} hours)", + result.backfill_duration_secs, + result.backfill_duration_secs / 3600.0 + ); + println!( + " total bytes written: {:.4} GB", + result.total_bytes_written as f64 / GB as f64 + ); + println!(); + println!(" DISTRIBUTION:"); + println!( + " old shard CV: {:.6} ({:.2}%)", + result.old_shard_cv, + result.old_shard_cv * 100.0 + ); + println!( + " new shard CV: {:.6} ({:.2}%)", + result.new_shard_cv, + result.new_shard_cv * 100.0 + ); + println!(); + } + + // Validate invariants. + println!("═══════════════════════════════════════════════════════════════════"); + println!("Invariant Checks"); + println!("═══════════════════════════════════════════════════════════════════"); + + let mut all_pass = true; + for (label, _params, result, _) in &results { + let storage_ok = (result.storage_amplification - 2.0).abs() < 0.01; + let dual_write_ok = (result.dual_write_amplification - 2.0).abs() < 0.01; + let cv_ok = result.old_shard_cv < 0.05 && result.new_shard_cv < 0.05; + + let status = |ok| if ok { "PASS" } else { "FAIL" }; + println!(" {}:", label); + println!( + " storage amplification == 2.0×: {} ({:.4}×)", + status(storage_ok), + result.storage_amplification + ); + println!( + " dual-write amplification == 2.0×: {} ({:.4}×)", + status(dual_write_ok), + result.dual_write_amplification + ); + println!( + " hash distribution CV < 5%: {} (old={:.4}%, new={:.4}%)", + status(cv_ok), + result.old_shard_cv * 100.0, + result.new_shard_cv * 100.0 + ); + + if !storage_ok || !dual_write_ok || !cv_ok { + all_pass = false; + } + } + + println!(); + if all_pass { + println!("All invariant checks PASSED. The 2× transient load caveat is confirmed."); + } else { + println!("Some invariant checks FAILED. Review results above."); + std::process::exit(1); + } +} diff --git a/crates/miroir-core/src/reshard.rs b/crates/miroir-core/src/reshard.rs index 0691932..dcfac96 100644 --- a/crates/miroir-core/src/reshard.rs +++ b/crates/miroir-core/src/reshard.rs @@ -72,6 +72,7 @@ impl TimeWindow { /// Resharding configuration (plan §13.1 + schedule window guard). #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ReshardingConfig { + #[serde(default = "default_true")] pub enabled: bool, #[serde(default = "default_backfill_concurrency")] pub backfill_concurrency: usize, diff --git a/crates/miroir-ctl/src/commands/reshard.rs b/crates/miroir-ctl/src/commands/reshard.rs index 884ad9b..4316c5d 100644 --- a/crates/miroir-ctl/src/commands/reshard.rs +++ b/crates/miroir-ctl/src/commands/reshard.rs @@ -1,15 +1,282 @@ +//! Reshard CLI command: start, status, and schedule window guard. + use clap::Subcommand; +use miroir_core::reshard::{check_window_now, ReshardingConfig, WindowGuardResult}; #[derive(Subcommand, Debug)] pub enum ReshardSubcommand { - /// Start a reshard operation - Start, - /// Show reshard status - Status, - /// Cancel an active reshard - Cancel, + /// Start an online resharding operation (plan §13.1). + /// + /// Creates a shadow index with the new shard count, backfills from the + /// live index, verifies, and swaps. Refuses to start outside the + /// configured schedule window unless --force is given. + Start { + /// Index UID to reshard. + #[arg(long)] + index: String, + + /// Target shard count. + #[arg(long)] + new_shards: u32, + + /// Backfill throttle (docs/sec). 0 = unlimited. + #[arg(long, default_value = "10000")] + throttle: u64, + + /// Named schedule window (from config). Pass "off-peak" or the + /// configured window name. The command refuses to start outside + /// this window unless --force is given. + #[arg(long)] + schedule_window: Option, + + /// Override schedule window guard — start resharding regardless + /// of the current time window. + #[arg(long)] + force: bool, + + /// Dry run: show what would happen without starting. + #[arg(long)] + dry_run: bool, + }, + + /// Check the status of an ongoing resharding operation. + Status { + /// Index UID to check. + #[arg(long)] + index: String, + }, } -pub async fn run(_cmd: ReshardSubcommand) -> Result<(), Box> { - Err("This command is not yet implemented. See bead miroir-qon for tracking.".into()) +pub async fn run(cmd: ReshardSubcommand) -> Result<(), Box> { + match cmd { + ReshardSubcommand::Start { + index, + new_shards, + throttle, + schedule_window, + force, + dry_run, + } => run_start(index, new_shards, throttle, schedule_window, force, dry_run).await, + ReshardSubcommand::Status { index } => run_status(index).await, + } +} + +async fn run_start( + index: String, + new_shards: u32, + throttle: u64, + schedule_window: Option, + force: bool, + dry_run: bool, +) -> Result<(), Box> { + let config = load_reshard_config()?; + + if !config.enabled { + return Err("Resharding is disabled in configuration (resharding.enabled = false)".into()); + } + + // Schedule window guard. + let guard = check_window_now(&config); + match &guard { + WindowGuardResult::Denied { utc_now, allowed } => { + if !force { + eprintln!("Error: resharding is not allowed at {}.", utc_now); + eprintln!("Allowed windows: {}", allowed.join(", ")); + eprintln!("Use --force to override (not recommended during peak load)."); + std::process::exit(1); + } + eprintln!( + "Warning: forcing resharding outside allowed window (now: {}, allowed: {})", + utc_now, + allowed.join(", ") + ); + } + WindowGuardResult::Allowed { window } => { + eprintln!("Schedule window check: within allowed window ({})", window); + } + WindowGuardResult::NoRestriction => { + eprintln!("Schedule window check: no restriction configured"); + } + } + + // Validate schedule_window argument against config. + if let Some(ref window_name) = schedule_window { + if !config.allowed_windows.is_empty() { + let found = config.allowed_windows.iter().any(|w| w == window_name); + if !found { + eprintln!( + "Warning: --schedule-window '{}' not found in config allowed_windows: [{}]", + window_name, + config.allowed_windows.join(", ") + ); + } + } + } + + if dry_run { + println!( + "Dry run: would reshard index '{}' to {} shards", + index, new_shards + ); + println!(" throttle: {} docs/sec", throttle); + println!(" force: {}", force); + println!(" schedule_window: {:?}", schedule_window); + println!(" window_guard: {:?}", guard); + println!( + " config.backfill_concurrency: {}", + config.backfill_concurrency + ); + println!( + " config.backfill_batch_size: {}", + config.backfill_batch_size + ); + println!(" config.verify_before_swap: {}", config.verify_before_swap); + println!( + " config.retain_old_index_hours: {}h", + config.retain_old_index_hours + ); + println!(); + println!("Phase plan:"); + println!(" 1. Shadow create: {index}__reshard_{new_shards}"); + println!(" 2. Dual-hash dual-write begins"); + println!( + " 3. Backfill (throttled to {throttle} docs/sec, concurrency {})", + config.backfill_concurrency + ); + println!(" 4. Verify PK-set and content hashes"); + println!(" 5. Alias swap"); + println!( + " 6. Cleanup (retain old for {}h)", + config.retain_old_index_hours + ); + return Ok(()); + } + + // TODO: Submit reshard job via admin API when proxy is implemented. + Err("Reshard start requires admin API (not yet connected). Use --dry-run to validate.".into()) +} + +async fn run_status(index: String) -> Result<(), Box> { + // TODO: Query reshard status via admin API when proxy is implemented. + let _ = index; + Err("Reshard status requires admin API (not yet connected).".into()) +} + +/// Load resharding config from the standard config path. +/// +/// Looks for `~/.config/miroir/config.toml` and extracts the +/// `[resharding]` section. Returns defaults if no config found. +fn load_reshard_config() -> Result> { + use std::fs; + use std::path::PathBuf; + + let config_path = dirs::config_dir() + .map(|d| d.join("miroir").join("config.toml")) + .unwrap_or_else(|| PathBuf::from("/dev/null")); + + if !config_path.exists() { + return Ok(ReshardingConfig::default()); + } + + let contents = fs::read_to_string(&config_path) + .map_err(|e| format!("Failed to read {}: {}", config_path.display(), e))?; + + let full: toml::Value = toml::from_str(&contents) + .map_err(|e| format!("Invalid TOML in {}: {}", config_path.display(), e))?; + + let resharding = full + .get("resharding") + .cloned() + .unwrap_or(toml::Value::Table(toml::map::Map::new())); + + let config: ReshardingConfig = resharding + .try_into() + .map_err(|e| format!("Invalid [resharding] config: {}", e))?; + + Ok(config) +} + +#[cfg(test)] +mod tests { + use miroir_core::reshard::{check_window, ReshardingConfig, WindowGuardResult}; + + #[test] + fn start_refused_outside_window() { + let config = ReshardingConfig { + allowed_windows: vec!["02:00-06:00".into()], + ..Default::default() + }; + let guard = check_window(720, &config); + assert!(matches!(guard, WindowGuardResult::Denied { .. })); + } + + #[test] + fn start_allowed_inside_window() { + let config = ReshardingConfig { + allowed_windows: vec!["02:00-06:00".into()], + ..Default::default() + }; + let guard = check_window(180, &config); + assert!(matches!(guard, WindowGuardResult::Allowed { .. })); + } + + #[test] + fn start_no_restriction_when_unconfigured() { + let config = ReshardingConfig::default(); + let guard = check_window(720, &config); + assert!(matches!(guard, WindowGuardResult::NoRestriction)); + } + + #[test] + fn wrap_midnight_window() { + let config = ReshardingConfig { + allowed_windows: vec!["22:00-06:00".into()], + ..Default::default() + }; + assert!(matches!( + check_window(1380, &config), + WindowGuardResult::Allowed { .. } + )); + assert!(matches!( + check_window(60, &config), + WindowGuardResult::Allowed { .. } + )); + assert!(matches!( + check_window(720, &config), + WindowGuardResult::Denied { .. } + )); + } + + #[test] + fn parse_resharding_config_from_toml() { + let toml = r#" +enabled = true +backfill_concurrency = 8 +backfill_batch_size = 500 +throttle_docs_per_sec = 5000 +verify_before_swap = true +retain_old_index_hours = 24 +allowed_windows = ["02:00-06:00", "22:00-23:30"] +"#; + let config: ReshardingConfig = toml::from_str(toml).unwrap(); + assert!(config.enabled); + assert_eq!(config.backfill_concurrency, 8); + assert_eq!(config.backfill_batch_size, 500); + assert_eq!(config.throttle_docs_per_sec, 5000); + assert_eq!(config.retain_old_index_hours, 24); + assert_eq!(config.allowed_windows.len(), 2); + } + + #[test] + fn parse_resharding_config_defaults() { + let toml = ""; + let config: ReshardingConfig = toml::from_str(toml).unwrap(); + assert!(config.enabled); + assert_eq!(config.backfill_concurrency, 4); + assert_eq!(config.backfill_batch_size, 1000); + assert_eq!(config.throttle_docs_per_sec, 0); + assert!(config.verify_before_swap); + assert_eq!(config.retain_old_index_hours, 48); + assert!(config.allowed_windows.is_empty()); + } } diff --git a/crates/miroir-ctl/tests/window_guard.rs b/crates/miroir-ctl/tests/window_guard.rs new file mode 100644 index 0000000..69fc0ed --- /dev/null +++ b/crates/miroir-ctl/tests/window_guard.rs @@ -0,0 +1,129 @@ +//! Integration tests for the reshard CLI schedule window guard (P12.OP3). +//! +//! These tests exercise the full CLI binary as a subprocess to confirm that +//! the window guard correctly rejects resharding outside configured windows +//! and that --force overrides the guard. + +use std::fs; +use std::process::Command; +use tempfile::TempDir; + +fn bin() -> String { + env!("CARGO_BIN_EXE_miroir-ctl").to_string() +} + +/// Write a TOML config with the given resharding section into a temp dir, +/// returning the TempDir (kept alive for the subprocess). +fn write_config(resharding_toml: &str) -> TempDir { + let tmp = TempDir::new().unwrap(); + let config_dir = tmp.path().join("miroir"); + fs::create_dir_all(&config_dir).unwrap(); + fs::write(config_dir.join("config.toml"), resharding_toml).unwrap(); + tmp +} + +fn run_reshard(tmp: &TempDir, extra_args: &[&str]) -> std::process::Output { + Command::new(bin()) + .env("XDG_CONFIG_HOME", tmp.path()) + .env("MIROIR_ADMIN_API_KEY", "test-key") + .args([ + "reshard", + "start", + "--index", + "test-idx", + "--new-shards", + "128", + "--dry-run", + ]) + .args(extra_args) + .output() + .unwrap() +} + +#[test] +fn rejected_outside_configured_window() { + // Use a 1-minute window in the middle of the night — very unlikely to match now. + let tmp = write_config( + r#"[resharding] +enabled = true +allowed_windows = ["03:42-03:43"]"#, + ); + + let output = run_reshard(&tmp, &[]); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert!( + !output.status.success(), + "CLI should fail outside window. stderr: {stderr}" + ); + assert!( + stderr.contains("not allowed") || stderr.contains("Error"), + "stderr should mention rejection: {stderr}" + ); +} + +#[test] +fn force_overrides_window_guard() { + let tmp = write_config( + r#"[resharding] +enabled = true +allowed_windows = ["03:42-03:43"]"#, + ); + + let output = run_reshard(&tmp, &["--force"]); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert!( + output.status.success(), + "CLI should succeed with --force. stderr: {stderr}" + ); + assert!( + stderr.contains("forcing resharding outside"), + "stderr should warn about force override: {stderr}" + ); +} + +#[test] +fn no_windows_allows_any_time() { + let tmp = write_config( + r#"[resharding] +enabled = true"#, + ); + + let output = run_reshard(&tmp, &[]); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert!( + output.status.success(), + "CLI should succeed when no windows configured. stderr: {stderr}" + ); + assert!( + stderr.contains("no restriction"), + "stderr should note no restriction: {stderr}" + ); + assert!( + stdout.contains("Dry run"), + "stdout should show dry run plan: {stdout}" + ); +} + +#[test] +fn disabled_config_rejects_even_with_no_windows() { + let tmp = write_config( + r#"[resharding] +enabled = false"#, + ); + + let output = run_reshard(&tmp, &[]); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert!( + !output.status.success(), + "CLI should fail when resharding disabled. stderr: {stderr}" + ); + assert!( + stderr.contains("disabled"), + "stderr should mention disabled: {stderr}" + ); +} diff --git a/docs/benchmarks/resharding-load.md b/docs/benchmarks/resharding-load.md new file mode 100644 index 0000000..c08aac2 --- /dev/null +++ b/docs/benchmarks/resharding-load.md @@ -0,0 +1,97 @@ +# Resharding Load Benchmark Results + +Plan §15 Open Problem #3: Empirical validation of the 2× transient load caveat. + +## Summary + +Online resharding (§13.1) creates a shadow index alongside the live index, backfills documents, verifies, then swaps aliases. The plan states that this "doubles transient storage and write load." This benchmark confirms that estimate using the actual routing code with synthetic corpora. + +**Conclusion: Storage amplification is exactly 2.0× and dual-write amplification is exactly 2.0× across all tested scenarios.** However, peak write amplification (dual-write + backfill) depends heavily on the backfill throttle rate relative to the incoming write rate and can reach 500× or more if the backfill runs unthrottled against a low-write corpus. + +## Test Matrix + +| Doc size | Corpus | Write rate | RG | RF | Nodes/group | Old→New shards | Backfill throttle | +|----------|--------|------------|----|----|-------------|----------------|-------------------| +| 1 KB | 10 GB | 100 dps | 2 | 1 | 3 | 64→128 | 10,000 dps | +| 10 KB | 100 GB | 1,000 dps | 2 | 2 | 4 | 64→128 | 10,000 dps | +| 1 MB | 1 TB | 10 dps | 2 | 1 | 4 | 64→128 | 5,000 dps | + +## Results + +### Scenario 1: Small documents, moderate corpus + +| Metric | Value | +|--------|-------| +| Total documents | 10,485,760 | +| Normal storage (steady-state) | 20.00 GB | +| Peak storage (resharding) | 40.00 GB | +| **Storage amplification** | **2.00×** | +| Per-node normal | 3.91 GB | +| Per-node peak | 8.05 GB | +| Normal write rate | 200 writes/sec | +| Dual-write rate | 400 writes/sec (2.0×) | +| Peak write rate (bf+dw) | 20,400 writes/sec (102.0×) | +| Backfill duration | 17.5 min | +| Total bytes written | 20.40 GB | +| Old shard CV | 0.21% | +| New shard CV | 0.33% | + +### Scenario 2: Medium documents, large corpus, RF=2 + +| Metric | Value | +|--------|-------| +| Total documents | 10,485,760 | +| Normal storage (steady-state) | 200.00 GB | +| Peak storage (resharding) | 400.00 GB | +| **Storage amplification** | **2.00×** | +| Per-node normal | 62.51 GB | +| Per-node peak | 118.73 GB | +| Normal write rate | 4,000 writes/sec | +| Dual-write rate | 8,000 writes/sec (2.0×) | +| Peak write rate (bf+dw) | 48,000 writes/sec (12.0×) | +| Backfill duration | 17.5 min | +| Total bytes written | 480.00 GB | +| Old shard CV | 0.21% | +| New shard CV | 0.33% | + +### Scenario 3: Large blobs, very large corpus + +| Metric | Value | +|--------|-------| +| Total documents | 1,048,576 | +| Normal storage (steady-state) | 2,048.00 GB | +| Peak storage (resharding) | 4,096.00 GB | +| **Storage amplification** | **2.00×** | +| Per-node normal | 319.90 GB | +| Per-node peak | 647.67 GB | +| Normal write rate | 20 writes/sec | +| Dual-write rate | 40 writes/sec (2.0×) | +| Peak write rate (bf+dw) | 10,040 writes/sec (502.0×) | +| Backfill duration | 3.5 min | +| Total bytes written | 2,056.19 GB | +| Old shard CV | 0.66% | +| New shard CV | 1.04% | + +## Invariant Verification + +| Invariant | S1 | S2 | S3 | +|-----------|-----|-----|-----| +| Storage amplification == 2.0× | PASS (2.00) | PASS (2.00) | PASS (2.00) | +| Dual-write amplification == 2.0× | PASS (2.00) | PASS (2.00) | PASS (2.00) | +| Hash distribution CV < 5% | PASS (0.21%, 0.33%) | PASS (0.21%, 0.33%) | PASS (0.66%, 1.04%) | + +## Operator Guidance + +1. **Storage**: Reserve exactly 2× your corpus size (× RG) before starting a reshard. No exceptions. +2. **Write load during dual-write**: Expect exactly 2× your normal write throughput. Plan headroom accordingly. +3. **Peak write load**: The backfill throttle dominates peak write load. Set `throttle_docs_per_sec` conservatively — aim for peak total writes ≤ 3× normal. Formula: `backfill_throttle_dps × RF × RG + write_rate × 2 × RF × RG`. +4. **Backfill duration**: At 10K docs/sec, a 10M doc corpus takes ~17 minutes. Scale linearly. +5. **Schedule window**: Use `resharding.allowed_windows` to restrict resharding to off-peak hours. The CLI refuses to start outside configured windows unless `--force` is given. + +## Methodology + +The simulation builds a synthetic topology matching the test parameters, then iterates every document in the corpus, routing it through the real `shard_for_key` (XxHash64) and `assign_shard_in_group` (rendezvous hash) functions. Storage and write amplification are computed from the resulting shard assignments. + +Run with: `cargo run --bin bench-reshard-load` + +Last run: 2026-04-18