From 65ad0bc212f9a532cedde83b75ac0a1f513de8e2 Mon Sep 17 00:00:00 2001 From: jedarden Date: Fri, 10 Jul 2026 17:25:56 -0400 Subject: [PATCH] feat(reshard): seed real source-index doc count in start_backfill (bf-2tddo) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the legacy `total_documents: 0` placeholder on the in-memory BackfillProgress in `start_backfill` with the real source-index document count via `compute_source_document_count` (the bf-2ynu5 helper that delegates to `crate::index_stats::aggregate_index_stats`). The progress ratio now has a non-zero denominator from the very first shard. A stats-query failure must never abort a reshard over a mere progress signal, so the call maps an error to `0` via `.unwrap_or(0)` โ€” `advance_backfill`'s per-shard accumulation remains the legacy fallback that fills the denominator in incrementally. In-memory only: no change to the persisted ReshardOperation and no `upfront_total_known` reconciliation field in this child (those are children 2 and 3). Co-Authored-By: Claude --- crates/miroir-core/src/reshard/executor.rs | 32 ++++++++++++++++------ 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/crates/miroir-core/src/reshard/executor.rs b/crates/miroir-core/src/reshard/executor.rs index 8f63626..aea1772 100644 --- a/crates/miroir-core/src/reshard/executor.rs +++ b/crates/miroir-core/src/reshard/executor.rs @@ -511,11 +511,22 @@ impl ReshardExecutor { /// Phase 3: Start backfill from live to shadow. async fn start_backfill(&self, state: &mut ReshardState) -> Result<()> { - // Get total document count for progress tracking - // TODO: Query nodes for document counts + // Backfill denominator (plan ยง13.1 step 3, bf-2tddo): query the source + // index's real document count up front via the shared `index_stats` + // aggregation, so the progress ratio has a non-zero denominator from the + // very first shard instead of the legacy `0` placeholder. + // + // Progress-reporting only: a stats failure must never abort a reshard + // over a mere progress signal, so on error the denominator falls back to + // `0` and `advance_backfill`'s per-shard accumulation fills it in + // incrementally as the legacy fallback. + let total_documents = self + .compute_source_document_count(&state.index_uid) + .await + .unwrap_or(0); state.backfill_progress = BackfillProgress { - total_documents: 0, // Will be updated + total_documents, processed_documents: 0, current_shard: Some(0), last_cursor: None, @@ -523,6 +534,7 @@ impl ReshardExecutor { tracing::info!( index = %state.index_uid, + total_documents, "Started backfill" ); @@ -542,12 +554,16 @@ impl ReshardExecutor { /// The aggregation is infallible from the caller's perspective โ€” a node that /// fails to respond (network / non-2xx / parse error) is logged and skipped, /// and the count is `0` only if every node fails or reports zero โ€” but this - /// returns [`Result`] to match the executor's method conventions and leave - /// the `start_backfill` caller (wired in a sibling bead) free to propagate a - /// future failure mode. + /// returns [`Result`] to match the executor's method conventions: the only + /// caller (`start_backfill`, wired in bf-2tddo) maps an error to a `0` + /// denominator so a stats failure never aborts a reshard, after which the + /// per-shard accumulation in `advance_backfill` fills the denominator in + /// incrementally as the legacy fallback. /// - /// Pure additive helper: this method is *not* called from the reshard state - /// machine in this bead. + /// Wired into `start_backfill` (bf-2tddo) as the backfill denominator; it + /// was introduced as a pure additive helper in bf-2ynu5 and is now the + /// source of the non-zero `total_documents` that backfill progress divides + /// by. pub async fn compute_source_document_count(&self, index_uid: &str) -> Result { let stats = crate::index_stats::aggregate_index_stats( &self.http_client,