Phase 10: Live evolution observatory - evolver live.json feed + observatory page

Evolver writes live.json to R2 every cycle. Observatory page polls and
renders live feed + lineage tree + meta shift chart.

- Added ACB_R2_UPLOAD_ENABLED env var to enable automatic R2 upload during run loop
- CycleState tracks real-time evolution cycle status (generation, phase, candidate, validation, evaluation)
- Export() now includes cycle info when cycleState is provided
- runCycle() integrated with live observatory exports at each phase transition
- exportLiveQuiet() for mid-cycle status updates without verbose logging
- Fixed function signature mismatches for exportLiveQuiet calls

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

.needle-predispatch-sha

@@ -1 +1 @@
-17965d8207614a0f039062ed5fe43b945f7f9f08
+02338375440c4ead8fd7156f034ee29a334daef3

cmd/acb-evolver/internal/live/cycle.go

@@ -0,0 +1,299 @@
+// Package live provides real-time cycle state tracking for the evolution observatory.
+package live
+
+import (
+	"fmt"
+	"sync"
+	"time"
+)
+
+// CycleState tracks the current evolution cycle status in real-time.
+// This is updated throughout the cycle and exported to live.json.
+type CycleState struct {
+	mu               sync.RWMutex
+	Generation       int
+	StartedAt        time.Time
+	Phase            string // generating, validating, evaluating, promoting, idle
+	CandidateID      string
+	CandidateIsland  string
+	CandidateLang    string
+	ParentIDs        []string
+	Validation       *CycleValidation
+	Evaluation       *CycleEvaluation
+	PromotionReason  string // Set when promoted/rejected
+	CommunityHint    string // Community hint that influenced this candidate
+}
+
+// CycleValidation tracks validation stage progress.
+type CycleValidation struct {
+	SyntaxPassed   bool
+	SyntaxTimeMs   int
+	SchemaPassed   bool
+	SchemaTimeMs   int
+	SmokePassed    bool
+	SmokeTimeMs    int
+	LastErrorStage string
+	LastError      string
+}
+
+// CycleEvaluation tracks arena evaluation progress.
+type CycleEvaluation struct {
+	MatchesTotal  int
+	MatchesPlayed int
+	Results       []CycleMatchResult
+}
+
+// CycleMatchResult is a single evaluation match result.
+type CycleMatchResult struct {
+	Opponent string
+	Won      bool
+	Score    string // e.g. "5-1"
+}
+
+// NewCycleState creates a new cycle state tracker.
+func NewCycleState() *CycleState {
+	return &CycleState{
+		Phase: "idle",
+	}
+}
+
+// SetPhase updates the current phase and timestamp.
+func (c *CycleState) SetPhase(phase string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.Phase = phase
+	if phase == "generating" && c.StartedAt.IsZero() {
+		c.StartedAt = time.Now().UTC()
+	}
+}
+
+// SetGeneration sets the current generation number.
+func (c *CycleState) SetGeneration(gen int) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.Generation = gen
+}
+
+// SetCandidate sets the current candidate being evaluated.
+func (c *CycleState) SetCandidate(id, island, lang string, parents []string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.CandidateID = id
+	c.CandidateIsland = island
+	c.CandidateLang = lang
+	c.ParentIDs = parents
+}
+
+// SetValidationSyntax records the result of syntax validation.
+func (c *CycleState) SetValidationSyntax(passed bool, timeMs int) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.Validation == nil {
+		c.Validation = &CycleValidation{}
+	}
+	c.Validation.SyntaxPassed = passed
+	c.Validation.SyntaxTimeMs = timeMs
+	if !passed {
+		c.Validation.LastErrorStage = "syntax"
+	}
+}
+
+// SetValidationSchema records the result of schema validation.
+func (c *CycleState) SetValidationSchema(passed bool, timeMs int) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.Validation == nil {
+		c.Validation = &CycleValidation{}
+	}
+	c.Validation.SchemaPassed = passed
+	c.Validation.SchemaTimeMs = timeMs
+	if !passed {
+		c.Validation.LastErrorStage = "schema"
+	}
+}
+
+// SetValidationSmoke records the result of smoke test validation.
+func (c *CycleState) SetValidationSmoke(passed bool, timeMs int) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.Validation == nil {
+		c.Validation = &CycleValidation{}
+	}
+	c.Validation.SmokePassed = passed
+	c.Validation.SmokeTimeMs = timeMs
+	if !passed {
+		c.Validation.LastErrorStage = "smoke"
+	}
+}
+
+// SetValidationError records a validation error.
+func (c *CycleState) SetValidationError(stage, errMsg string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.Validation == nil {
+		c.Validation = &CycleValidation{}
+	}
+	c.Validation.LastErrorStage = stage
+	c.Validation.LastError = errMsg
+}
+
+// StartEvaluation initializes the evaluation phase.
+func (c *CycleState) StartEvaluation(totalMatches int) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.Evaluation = &CycleEvaluation{
+		MatchesTotal:  totalMatches,
+		MatchesPlayed: 0,
+		Results:       make([]CycleMatchResult, 0, totalMatches),
+	}
+}
+
+// AddEvaluationResult adds a match result to the evaluation.
+func (c *CycleState) AddEvaluationResult(opponent string, won bool, score string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.Evaluation == nil {
+		return
+	}
+	c.Evaluation.Results = append(c.Evaluation.Results, CycleMatchResult{
+		Opponent: opponent,
+		Won:      won,
+		Score:    score,
+	})
+	c.Evaluation.MatchesPlayed = len(c.Evaluation.Results)
+}
+
+// SetPromotionResult sets the final promotion decision.
+func (c *CycleState) SetPromotionResult(reason string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.PromotionReason = reason
+}
+
+// SetCommunityHint sets the community hint that influenced this candidate.
+func (c *CycleState) SetCommunityHint(hint string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.CommunityHint = hint
+}
+
+// SetPhaseInfo sets all candidate info at once (simplified interface).
+func (c *CycleState) SetPhaseInfo(phase, candidateID, island, lang string, parents []string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.Phase = phase
+	c.CandidateID = candidateID
+	c.CandidateIsland = island
+	c.CandidateLang = lang
+	c.ParentIDs = parents
+	if phase == "generating" && c.StartedAt.IsZero() {
+		c.StartedAt = time.Now().UTC()
+	}
+}
+
+// SetArenaResult records the final arena result.
+func (c *CycleState) SetArenaResult(wins, losses, draws, errors int, winRate float64) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.Evaluation == nil {
+		c.Evaluation = &CycleEvaluation{
+			MatchesTotal:  wins + losses + draws + errors,
+			Results:      make([]CycleMatchResult, 0),
+		}
+	}
+	c.Evaluation.MatchesPlayed = c.Evaluation.MatchesTotal
+	// Add a summary result
+	c.Evaluation.Results = append(c.Evaluation.Results, CycleMatchResult{
+		Opponent: "arena",
+		Won:      wins > losses,
+		Score:    fmt.Sprintf("%d-%d-%d", wins, losses, draws),
+	})
+}
+
+// Reset clears the cycle state (use between cycles).
+func (c *CycleState) Reset() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.Generation = 0
+	c.StartedAt = time.Time{}
+	c.Phase = "idle"
+	c.CandidateID = ""
+	c.CandidateIsland = ""
+	c.CandidateLang = ""
+	c.ParentIDs = nil
+	c.Validation = nil
+	c.Evaluation = nil
+	c.PromotionReason = ""
+	c.CommunityHint = ""
+}
+
+// ToCycleInfo converts the cycle state to an exportable CycleInfo.
+func (c *CycleState) ToCycleInfo() *CycleInfo {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.Phase == "idle" || c.Generation == 0 {
+		return nil
+	}
+
+	info := &CycleInfo{
+		Generation: c.Generation,
+		StartedAt:  c.StartedAt.UTC().Format(time.RFC3339),
+		Phase:      c.Phase,
+	}
+
+	if c.CandidateID != "" {
+		info.Candidate = &Candidate{
+			ID:       c.CandidateID,
+			Island:   c.CandidateIsland,
+			Language: c.CandidateLang,
+		}
+
+		// Add parents
+		if len(c.ParentIDs) > 0 {
+			info.Candidate.Parents = make([]ParentInfo, len(c.ParentIDs))
+			for i, pid := range c.ParentIDs {
+				info.Candidate.Parents[i] = ParentInfo{ID: pid}
+			}
+		}
+
+		// Add validation status
+		if c.Validation != nil {
+			info.Candidate.Validation = &ValidationStatus{
+				Syntax: &StageResult{
+					Passed:  c.Validation.SyntaxPassed,
+					TimeMs:  c.Validation.SyntaxTimeMs,
+					Error:   c.Validation.LastError,
+				},
+				Schema: &StageResult{
+					Passed:  c.Validation.SchemaPassed,
+					TimeMs:  c.Validation.SchemaTimeMs,
+				},
+				Smoke: &StageResult{
+					Passed:  c.Validation.SmokePassed,
+					TimeMs:  c.Validation.SmokeTimeMs,
+				},
+			}
+		}
+
+		// Add evaluation status
+		if c.Evaluation != nil {
+			info.Candidate.Evaluation = &EvaluationStatus{
+				MatchesTotal:  c.Evaluation.MatchesTotal,
+				MatchesPlayed: c.Evaluation.MatchesPlayed,
+			}
+			if len(c.Evaluation.Results) > 0 {
+				info.Candidate.Evaluation.Results = make([]MatchResult, len(c.Evaluation.Results))
+				for i, r := range c.Evaluation.Results {
+					info.Candidate.Evaluation.Results[i] = MatchResult{
+						Opponent: r.Opponent,
+						Won:      r.Won,
+						Score:    r.Score,
+					}
+				}
+			}
+		}
+	}
+
+	return info
+}

cmd/acb-evolver/internal/live/exporter.go

@@ -151,13 +151,19 @@ type LiveData struct {
 }
 
 // Export queries the programs database and builds the current evolution state.
-func Export(ctx context.Context, db *sql.DB) (*LiveData, error) {
+// If cycleState is provided, it includes the current cycle status.
+func Export(ctx context.Context, db *sql.DB, cycleState *CycleState) (*LiveData, error) {
 	data := &LiveData{
 		UpdatedAt: time.Now().UTC().Format(time.RFC3339),
 		Islands:   make(map[string]IslandStat),
 		Totals:    Totals{},
 	}
 
+	// Add cycle info if available
+	if cycleState != nil {
+		data.Cycle = cycleState.ToCycleInfo()
+	}
+
 	if err := fillIslandStats(ctx, db, data); err != nil {
 		return nil, err
 	}

cmd/acb-evolver/main.go

@@ -704,7 +704,8 @@ func runLiveExport(ctx context.Context, db *sql.DB, args []string) {
 		os.Exit(1)
 	}
 
-	data, err := live.Export(ctx, db)
+	// No active cycle state for manual export
+	data, err := live.Export(ctx, db, nil)
 	if err != nil {
 		log.Fatalf("live-export: %v", err)
 	}

cmd/acb-evolver/run.go

@@ -125,7 +125,7 @@ func DefaultRunConfig() RunConfig {
 		EncryptionKey:     os.Getenv("ACB_ENCRYPTION_KEY"),
 		UseNsjail:         true,
 		LiveExportPath:    envOrDefault("ACB_EVOLUTION_OUT", "evolution/live.json"),
-		UploadR2:          false,
+		UploadR2:          envOrDefault("ACB_R2_UPLOAD_ENABLED", "false") == "true",
 		DeclarativeConfigRepo:    envOrDefault("ACB_DECLARATIVE_CONFIG_REPO", "https://forgejo.ardenone.com/infra/ardenone-cluster.git"),
 		DeclarativeConfigBranch:  envOrDefault("ACB_DECLARATIVE_CONFIG_BRANCH", "main"),
 		Languages:         []string{"go", "python", "rust", "typescript", "java", "php"},
@@ -227,6 +227,9 @@ func RunEvolutionLoop(ctx context.Context, dbURL string, args []string) {
 	// Stats
 	stats := RunStats{StartTime: time.Now()}
 
+	// Shared cycle state for live observatory
+	cycleState := live.NewCycleState()
+
 	// Setup signal handling for graceful shutdown
 	ctx, cancel := context.WithCancel(ctx)
 	sigCh := make(chan os.Signal, 1)
@@ -282,10 +285,13 @@ func RunEvolutionLoop(ctx context.Context, dbURL string, args []string) {
 		}
 
 		// Run one evolution cycle
-		promoted, err := runCycle(ctx, db, store, island, lang, cfg, rng, *verbose, *dryRun)
+		cycleState.SetGeneration(stats.Cycles + 1)
+		promoted, err := runCycle(ctx, db, store, island, lang, cfg, rng, *verbose, *dryRun, cycleState)
 		if err != nil {
 			log.Printf("Cycle failed: %v", err)
 			stats.Errors++
+			cycleState.SetPhase("idle")
+			exportLive(ctx, db, cfg, *verbose, cycleState)
 		}
 		if promoted {
 			stats.Promoted++
@@ -303,7 +309,8 @@ func RunEvolutionLoop(ctx context.Context, dbURL string, args []string) {
 		}
 
 		// Export live.json after each cycle
-		exportLive(ctx, db, cfg, *verbose)
+		cycleState.SetPhase("idle")
+		exportLive(ctx, db, cfg, *verbose, cycleState)
 
 		// Check for cross-pollination (§10.2: every 50 generations per island)
 		cpChecker := crosspoll.NewChecker(store, llm.NewClient(cfg.LLMURL, ""), rng)
@@ -375,7 +382,7 @@ func selectNextIsland(lastEvolved map[string]time.Time, cooldown time.Duration,
 
 // runCycle executes one complete evolution cycle for the given island.
 func runCycle(ctx context.Context, db *sql.DB, store *evolverdb.Store,
-	island, lang string, cfg RunConfig, rng *rand.Rand, verbose, dryRun bool) (bool, error) {
+	island, lang string, cfg RunConfig, rng *rand.Rand, verbose, dryRun bool, cycleState *live.CycleState) (bool, error) {
 
 	// 1. Load programs from the island
 	programs, err := store.ListByIsland(ctx, island)
@@ -411,12 +418,24 @@ func runCycle(ctx context.Context, db *sql.DB, store *evolverdb.Store,
 	}
 	generation := maxGen + 1
 
+	// Set up cycle state for live observatory
+	cycleState.SetGeneration(generation)
+	cycleState.SetPhase("generating")
+	exportLiveQuiet(ctx, db, cfg, cycleState)
+
 	// 5. Generate candidate with retry loop
 	var programID int64
 	var code string
 	var program *evolverdb.Program
 	var report *validator.Report
 
+	// Build parent info for cycle state (with ratings)
+	parentInfos := make([]string, len(parents))
+	for i, p := range parents {
+		parentInfos[i] = fmt.Sprintf("%s-%d", island, p.ID)
+	}
+	cycleState.SetCandidate(fmt.Sprintf("%s-%d", lang, generation), island, lang, parentInfos)
+
 	for retry := 0; retry <= cfg.MaxRetries; retry++ {
 		if retry > 0 && verbose {
 			log.Printf("  Retry %d/%d with error feedback...", retry, cfg.MaxRetries)
@@ -480,13 +499,32 @@ func runCycle(ctx context.Context, db *sql.DB, store *evolverdb.Store,
 		valCfg.UseNsjail = cfg.UseNsjail
 
 		report, err = validator.Validate(ctx, code, lang, result.Best.Code, valCfg)
+			cycleState.SetPhase("validating")
+			exportLiveQuiet(ctx, db, cfg, cycleState)
 		if err != nil {
+				cycleState.SetValidationError("infrastructure", err.Error())
 			log.Printf("Validation infrastructure error: %v", err)
 			store.Delete(ctx, programID)
 			programID = 0
 			continue
 		}
 
+			// Track validation results in cycle state
+			for _, stage := range report.Stages {
+				timeMs := int(stage.Duration.Milliseconds())
+				switch stage.Stage {
+				case "syntax":
+					cycleState.SetValidationSyntax(stage.Passed, timeMs)
+				case "schema":
+					cycleState.SetValidationSchema(stage.Passed, timeMs)
+				case "smoke":
+					cycleState.SetValidationSmoke(stage.Passed, timeMs)
+				}
+				if !stage.Passed && stage.Error != "" {
+					cycleState.SetValidationError(string(stage.Stage), stage.Error)
+				}
+			}
+			exportLiveQuiet(ctx, db, cfg, cycleState)
 		// Log validation result
 		valLog := &evolverdb.ValidationLog{
 			Island:    island,
@@ -530,6 +568,8 @@ func runCycle(ctx context.Context, db *sql.DB, store *evolverdb.Store,
 	}
 
 	// 6. Run arena evaluation
+	cycleState.SetPhase("evaluating")
+	exportLiveQuiet(ctx, db, cfg, cycleState)
 	arenaCfg := arena.DefaultConfig()
 	arenaCfg.EncryptionKey = cfg.EncryptionKey
 	a := arena.New(db, arenaCfg)
@@ -761,8 +801,8 @@ Focus on fixing the specific error above while maintaining all required function
 }
 
 // exportLive exports the evolution state to live.json.
-func exportLive(ctx context.Context, db *sql.DB, cfg RunConfig, verbose bool) {
-	data, err := live.Export(ctx, db)
+func exportLive(ctx context.Context, db *sql.DB, cfg RunConfig, verbose bool, cycleState *live.CycleState) {
+	data, err := live.Export(ctx, db, cycleState)
 	if err != nil {
 		log.Printf("warn: live export failed: %v", err)
 		return
@@ -788,6 +828,24 @@ func exportLive(ctx context.Context, db *sql.DB, cfg RunConfig, verbose bool) {
 	}
 }
 
+// exportLiveQuiet is like exportLive but without verbose logging (for mid-cycle exports).
+func exportLiveQuiet(ctx context.Context, db *sql.DB, cfg RunConfig, cycleState *live.CycleState) {
+	data, err := live.Export(ctx, db, cycleState)
+	if err != nil {
+		return
+	}
+	_ = live.WriteFile(data, cfg.LiveExportPath)
+	if cfg.UploadR2 {
+		r2Cfg := live.R2ConfigFromEnv()
+		if r2Cfg.HasCredentials() {
+			r2Client, err := live.NewR2Client(r2Cfg)
+			if err == nil {
+				_ = r2Client.UploadLiveJSON(ctx, data)
+			}
+		}
+	}
+}
+
 // startRetirementTicker runs periodic retirement checks (§10.8).
 // This enforces the 7-day low-rating rule and 50-bot population cap.
 func startRetirementTicker(ctx context.Context, db *sql.DB, store *evolverdb.Store, cfg RunConfig, stats *RunStats, verbose bool) {