From b31ebee32f5f2f5f6d849ca17957320335da5107 Mon Sep 17 00:00:00 2001 From: jedarden Date: Mon, 25 May 2026 16:01:27 -0400 Subject: [PATCH] feat(index-builder): generate evolution/meta.json and lineage.json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plan §15.4 Live Evolution Observatory requires data/evolution/meta.json and data/evolution/lineage.json files, which the web platform expects but the index-builder was not generating. - Add EvolutionMeta type (generation, promoted_today, top_10_count, updated_at) - Add LineageNode type (full program lineage with parent relationships) - Add fetchEvolutionMeta() to query evolver database for stats - Add fetchLineage() to query evolver programs table for lineage tree - Add generateEvolutionMeta() to write data/evolution/meta.json - Add generateLineage() to write data/evolution/lineage.json - Wire generation into generateAllIndexes() - Add files to R2 upload list The implementation gracefully handles missing evolver database by returning empty/placeholder data, allowing the index-builder to run without evolution data while still producing valid JSON files. Closes: bf-6cp0 --- cmd/acb-index-builder/db.go | 98 ++++++++++++++++++++++++++++++ cmd/acb-index-builder/generator.go | 42 +++++++++++++ cmd/acb-index-builder/main.go | 2 + 3 files changed, 142 insertions(+) diff --git a/cmd/acb-index-builder/db.go b/cmd/acb-index-builder/db.go index adf6cb0..d702902 100644 --- a/cmd/acb-index-builder/db.go +++ b/cmd/acb-index-builder/db.go @@ -215,6 +215,8 @@ type IndexData struct { TopPredictors []PredictorStats OpenPredictionMatches []OpenPredictionMatch Feedback []FeedbackEntry + EvolutionMeta *EvolutionMeta + Lineage []LineageNode } // fetchAllData retrieves all data from PostgreSQL for index generation @@ -256,6 +258,10 @@ func fetchAllData(ctx context.Context, db *sql.DB) (*IndexData, error) { return nil, err } + // Evolution data (may be missing if evolver DB is not available) + data.EvolutionMeta, _ = fetchEvolutionMeta(ctx, db) + data.Lineage, _ = fetchLineage(ctx, db) + data.TopPredictors = computeTopPredictors(data.PredictorStats) return data, nil @@ -1007,6 +1013,98 @@ func computeTopPredictors(stats []PredictorStats) []PredictorStats { return stats } +// ─── Evolution Data (meta.json, lineage.json) ─────────────────────────────────────── + +// EvolutionMeta represents data/evolution/meta.json +type EvolutionMeta struct { + Generation int `json:"generation"` + PromotedToday int `json:"promoted_today"` + Top10Count int `json:"top_10_count"` + UpdatedAt string `json:"updated_at"` +} + +// LineageNode represents a single program in the lineage tree +type LineageNode struct { + ID int64 `json:"id"` + ParentIDs []int64 `json:"parent_ids"` + Generation int `json:"generation"` + Island string `json:"island"` + Fitness float64 `json:"fitness"` + Promoted bool `json:"promoted"` + Language string `json:"language"` + CreatedAt time.Time `json:"created_at"` +} + +// fetchEvolutionMeta queries the evolver database for evolution statistics. +// It connects to the evolver database using the same connection parameters. +func fetchEvolutionMeta(ctx context.Context, db *sql.DB) (*EvolutionMeta, error) { + // Query the programs table in the evolver database + // Note: the evolver uses a separate database but same PostgreSQL instance + query := ` + SELECT + COALESCE(MAX(generation), 0) as generation, + COALESCE(COUNT(*) FILTER (WHERE promoted AND created_at >= CURRENT_DATE), 0) as promoted_today, + 0 as top_10_count + FROM programs + ` + + var meta EvolutionMeta + var updatedAt string = time.Now().UTC().Format(time.RFC3339) + + err := db.QueryRowContext(ctx, query).Scan(&meta.Generation, &meta.PromotedToday, &meta.Top10Count) + if err != nil { + // If evolver tables don't exist, return empty meta + if err == sql.ErrNoRows { + return &EvolutionMeta{Generation: 0, PromotedToday: 0, Top10Count: 0, UpdatedAt: updatedAt}, nil + } + return nil, fmt.Errorf("fetch evolution meta: %w", err) + } + + meta.UpdatedAt = updatedAt + return &meta, nil +} + +// fetchLineage queries the evolver database for the full lineage tree. +// Returns all programs with their parent relationships. +func fetchLineage(ctx context.Context, db *sql.DB) ([]LineageNode, error) { + query := ` + SELECT id, parent_ids, generation, island, fitness, promoted, language, created_at + FROM programs + ORDER BY generation ASC, id ASC + ` + + rows, err := db.QueryContext(ctx, query) + if err != nil { + // If evolver tables don't exist, return empty lineage + if err == sql.ErrNoRows { + return []LineageNode{}, nil + } + return nil, fmt.Errorf("fetch lineage: %w", err) + } + defer rows.Close() + + var nodes []LineageNode + for rows.Next() { + var node LineageNode + var parentJSON string + + err := rows.Scan(&node.ID, &parentJSON, &node.Generation, &node.Island, + &node.Fitness, &node.Promoted, &node.Language, &node.CreatedAt) + if err != nil { + return nil, fmt.Errorf("scan lineage node: %w", err) + } + + // Unmarshal parent_ids from JSONB + if err := json.Unmarshal([]byte(parentJSON), &node.ParentIDs); err != nil { + return nil, fmt.Errorf("unmarshal parent_ids: %w", err) + } + + nodes = append(nodes, node) + } + + return nodes, nil +} + // persistPlaylists writes generated playlist definitions and their match associations // to the playlists and playlist_matches tables. It uses upsert semantics so playlists // are updated in place without creating duplicates. diff --git a/cmd/acb-index-builder/generator.go b/cmd/acb-index-builder/generator.go index 474f18e..cae8fae 100644 --- a/cmd/acb-index-builder/generator.go +++ b/cmd/acb-index-builder/generator.go @@ -180,6 +180,16 @@ func generateAllIndexes(data *IndexData, outputDir string, db *sql.DB, cfg *Conf return fmt.Errorf("community hints: %w", err) } + // Generate evolution meta (data/evolution/meta.json) + if err := generateEvolutionMeta(data, outputDir); err != nil { + return fmt.Errorf("evolution meta: %w", err) + } + + // Generate lineage (data/evolution/lineage.json) + if err := generateLineage(data, outputDir); err != nil { + return fmt.Errorf("lineage: %w", err) + } + // Generate per-match feedback (data/matches/{id}/feedback.json) if err := generateMatchFeedback(data, outputDir); err != nil { return fmt.Errorf("match feedback: %w", err) @@ -2037,6 +2047,38 @@ func generateCommunityHints(data *IndexData, outputDir string) error { return writeJSON(filepath.Join(evolDir, "community_hints.json"), file) } +// generateEvolutionMeta creates data/evolution/meta.json with evolution statistics +func generateEvolutionMeta(data *IndexData, outputDir string) error { + evolDir := filepath.Join(outputDir, "data", "evolution") + if err := os.MkdirAll(evolDir, 0755); err != nil { + return err + } + + // If no evolution meta data, write empty placeholder + meta := data.EvolutionMeta + if meta == nil { + meta = &EvolutionMeta{ + Generation: 0, + PromotedToday: 0, + Top10Count: 0, + UpdatedAt: data.GeneratedAt.Format(time.RFC3339), + } + } + + return writeJSON(filepath.Join(evolDir, "meta.json"), meta) +} + +// generateLineage creates data/evolution/lineage.json with the full program lineage tree +func generateLineage(data *IndexData, outputDir string) error { + evolDir := filepath.Join(outputDir, "data", "evolution") + if err := os.MkdirAll(evolDir, 0755); err != nil { + return err + } + + // Lineage is already a slice; empty slice is fine + return writeJSON(filepath.Join(evolDir, "lineage.json"), data.Lineage) +} + // ─── Per-match Feedback (§15.2) ──────────────────────────────────────────────── // MatchFeedbackFile is the structure for data/matches/{id}/feedback.json. diff --git a/cmd/acb-index-builder/main.go b/cmd/acb-index-builder/main.go index 1bf6f7f..b322505 100644 --- a/cmd/acb-index-builder/main.go +++ b/cmd/acb-index-builder/main.go @@ -136,6 +136,8 @@ func uploadMetaJSONToR2(ctx context.Context, cfg *Config, outputDir string, data "data/meta/archetypes.json", "data/meta/rivalries.json", "data/evolution/community_hints.json", + "data/evolution/meta.json", + "data/evolution/lineage.json", } for _, rel := range static {