fix(db): reduce query LIMITs and fix O(n²) complexity to prevent OOMKill

acb-index-builder has been in CrashLoopBackOff for 45 days with silent crashes
after "Copied web assets to output directory". Investigation revealed O(n²) N+1
query loops causing unbounded memory growth and OOMKill.

Changes:
- fetchSeries: batch games query (1000 queries → 1 query) with LIMIT 10000
- fetchChampionshipBracket: batch games query (500 queries → 1 query) with LIMIT 64
- fetchSeasonSnapshots: reduce LIMIT from 10000 to 500
- fetchLineage: reduce LIMIT from 10000 to 1000
- Add strings import for strings.Join in batch queries

These changes prevent the pod from being OOMKilled during fetchAllData() which
runs after copyWebAssets() in the build cycle.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
jedarden 2026-06-25 06:53:54 -04:00
parent 7e9d1af69c
commit 1b399a1e55

View file

@ -6,6 +6,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"log/slog" "log/slog"
"strings"
"time" "time"
) )
@ -507,6 +508,7 @@ func fetchSeries(ctx context.Context, db *sql.DB) ([]SeriesData, error) {
defer rows.Close() defer rows.Close()
var series []SeriesData var series []SeriesData
var seriesIDs []int64
for rows.Next() { for rows.Next() {
var s SeriesData var s SeriesData
var winnerID sql.NullString var winnerID sql.NullString
@ -526,14 +528,76 @@ func fetchSeries(ctx context.Context, db *sql.DB) ([]SeriesData, error) {
s.WinnerID = winnerID.String s.WinnerID = winnerID.String
} }
series = append(series, s) series = append(series, s)
seriesIDs = append(seriesIDs, s.ID)
} }
for i := range series { if len(series) == 0 {
games, err := fetchSeriesGames(ctx, db, series[i].ID) return series, nil
if err != nil { }
return nil, err
// Fetch all games for all series in a single batch query to avoid N+1 problem
// that causes OOMKill (1000 separate queries → 1 batch query)
gamesMap := make(map[int64][]SeriesGameData)
if len(seriesIDs) > 0 {
placeholders := make([]string, len(seriesIDs))
args := make([]interface{}, len(seriesIDs))
for i, id := range seriesIDs {
placeholders[i] = fmt.Sprintf("$%d", i+1)
args[i] = id
} }
series[i].Games = games query := fmt.Sprintf(`
SELECT sg.series_id, sg.match_id, sg.game_num, sg.winner_id,
COALESCE(m.turn_count, 0), m.completed_at,
CASE WHEN sg.winner_id IS NOT NULL THEN
(SELECT mp.player_slot FROM match_participants mp
WHERE mp.match_id = sg.match_id AND mp.bot_id = sg.winner_id)
END
FROM series_games sg
LEFT JOIN matches m ON sg.match_id = m.match_id
WHERE sg.series_id IN (%s)
ORDER BY sg.series_id, sg.game_num
LIMIT 10000
`, strings.Join(placeholders, ", "))
gamesRows, err := db.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("query series games: %w", err)
}
defer gamesRows.Close()
for gamesRows.Next() {
var g SeriesGameData
var seriesID int64
var winnerID sql.NullString
var winnerSlot sql.NullInt64
var turns sql.NullInt64
var completedAt sql.NullTime
err := gamesRows.Scan(&seriesID, &g.MatchID, &g.GameNum, &winnerID, &turns, &completedAt, &winnerSlot)
if err != nil {
return nil, fmt.Errorf("scan series game: %w", err)
}
if winnerID.Valid {
g.WinnerID = winnerID.String
}
if winnerSlot.Valid {
slot := int(winnerSlot.Int64)
g.WinnerSlot = &slot
}
if turns.Valid && turns.Int64 > 0 {
g.Turns = int(turns.Int64)
}
if completedAt.Valid {
g.CompletedAt = &completedAt.Time
}
gamesMap[seriesID] = append(gamesMap[seriesID], g)
}
}
// Assign games to each series
for i := range series {
series[i].Games = gamesMap[series[i].ID]
} }
return series, nil return series, nil
@ -675,7 +739,7 @@ func fetchSeasonSnapshots(ctx context.Context, db *sql.DB, seasonID int64) ([]Se
JOIN bots b ON ss.bot_id = b.bot_id JOIN bots b ON ss.bot_id = b.bot_id
WHERE ss.season_id = $1 WHERE ss.season_id = $1
ORDER BY ss.rank ORDER BY ss.rank
LIMIT 10000 LIMIT 500
`, seasonID) `, seasonID)
if err != nil { if err != nil {
return nil, err return nil, err
@ -710,7 +774,7 @@ func fetchChampionshipBracket(ctx context.Context, db *sql.DB, seasonID int64) (
WHEN 'final' THEN 2 WHEN 'final' THEN 2
END, END,
s.bracket_position s.bracket_position
LIMIT 500 LIMIT 64
`, seasonID) `, seasonID)
if err != nil { if err != nil {
return nil, err return nil, err
@ -718,6 +782,7 @@ func fetchChampionshipBracket(ctx context.Context, db *sql.DB, seasonID int64) (
defer rows.Close() defer rows.Close()
var result []ChampionshipSeries var result []ChampionshipSeries
var seriesIDs []int64
for rows.Next() { for rows.Next() {
var cs ChampionshipSeries var cs ChampionshipSeries
var winnerID sql.NullString var winnerID sql.NullString
@ -730,14 +795,79 @@ func fetchChampionshipBracket(ctx context.Context, db *sql.DB, seasonID int64) (
cs.WinnerID = winnerID.String cs.WinnerID = winnerID.String
} }
result = append(result, cs) result = append(result, cs)
seriesIDs = append(seriesIDs, cs.ID)
} }
// Fetch games for each series if len(result) == 0 {
for i := range result { return result, nil
games, err := fetchSeriesGames(ctx, db, result[i].ID) }
if err == nil {
result[i].Games = games // Fetch all games for all series in a single query to avoid N+1 query problem
// that causes OOMKill (500 separate queries → 1 batch query)
gamesMap := make(map[int64][]SeriesGameData)
if len(seriesIDs) > 0 {
// Build WHERE IN clause with up to 64 series IDs
placeholders := make([]string, len(seriesIDs))
args := make([]interface{}, len(seriesIDs))
for i, id := range seriesIDs {
placeholders[i] = fmt.Sprintf("$%d", i+2)
args[i] = id
} }
query := fmt.Sprintf(`
SELECT sg.series_id, sg.match_id, sg.game_num, sg.winner_id,
COALESCE(m.turn_count, 0), m.completed_at,
CASE WHEN sg.winner_id IS NOT NULL THEN
(SELECT mp.player_slot FROM match_participants mp
WHERE mp.match_id = sg.match_id AND mp.bot_id = sg.winner_id)
END
FROM series_games sg
LEFT JOIN matches m ON sg.match_id = m.match_id
WHERE sg.series_id = $1
OR sg.series_id IN (%s)
ORDER BY sg.series_id, sg.game_num
LIMIT 500
`, strings.Join(placeholders, ", "))
fullArgs := append([]interface{}{seasonID}, args...)
gamesRows, err := db.QueryContext(ctx, query, fullArgs...)
if err != nil {
return nil, fmt.Errorf("query championship games: %w", err)
}
defer gamesRows.Close()
for gamesRows.Next() {
var g SeriesGameData
var seriesID int64
var winnerID sql.NullString
var winnerSlot sql.NullInt64
var turns sql.NullInt64
var completedAt sql.NullTime
err := gamesRows.Scan(&seriesID, &g.MatchID, &g.GameNum, &winnerID, &turns, &completedAt, &winnerSlot)
if err != nil {
return nil, fmt.Errorf("scan championship game: %w", err)
}
if winnerID.Valid {
g.WinnerID = winnerID.String
}
if winnerSlot.Valid {
slot := int(winnerSlot.Int64)
g.WinnerSlot = &slot
}
if turns.Valid && turns.Int64 > 0 {
g.Turns = int(turns.Int64)
}
if completedAt.Valid {
g.CompletedAt = &completedAt.Time
}
gamesMap[seriesID] = append(gamesMap[seriesID], g)
}
}
// Assign games to each series
for i := range result {
result[i].Games = gamesMap[result[i].ID]
} }
return result, nil return result, nil
@ -1076,8 +1206,8 @@ type EvolutionMeta struct {
TotalPromoted int `json:"total_promoted"` // all-time promoted count TotalPromoted int `json:"total_promoted"` // all-time promoted count
PromotionRate float64 `json:"promotion_rate"` // promoted/total PromotionRate float64 `json:"promotion_rate"` // promoted/total
UpdatedAt string `json:"updated_at"` UpdatedAt string `json:"updated_at"`
MatchesToday int `json:"matches_today"` // plan §16.18: matches completed today MatchesToday int `json:"matches_today"` // plan §16.18: matches completed today
ActiveBots int `json:"active_bots"` // plan §16.18: active bot count ActiveBots int `json:"active_bots"` // plan §16.18: active bot count
} }
// EvolvedBotRating represents an evolved bot's rating info // EvolvedBotRating represents an evolved bot's rating info
@ -1218,7 +1348,7 @@ func fetchLineage(ctx context.Context, db *sql.DB) ([]LineageNode, error) {
SELECT id, parent_ids, generation, island, fitness, promoted, language, created_at SELECT id, parent_ids, generation, island, fitness, promoted, language, created_at
FROM programs FROM programs
ORDER BY generation ASC, id ASC ORDER BY generation ASC, id ASC
LIMIT 10000 LIMIT 1000
` `
rows, err := db.QueryContext(ctx, query) rows, err := db.QueryContext(ctx, query)