feat(index-builder): bundle warm-set replays as static Pages assets

Per bead bf-3e60: instead of copying B2->R2 (warm cache), bundle warm-set
replays, thumbnails, cards, and evolution live.json directly into the Pages
deploy directory as static assets (dist/data/). This serves replays same-origin,
eliminating R2 dependency and 404 errors.

Changes:
- Add B2Client interface for testable B2 operations
- Add bundleWarmReplays(): copies replays/*.json.gz from B2 to dist/data/replays/
- Add bundleWarmThumbnails(): copies thumbnails/*.png from B2 to dist/data/thumbnails/
- Add bundleWarmCards(): copies cards/*.png from B2 to dist/data/cards/
- Add bundleEvolutionLive(): copies evolution/live.json from B2 to dist/data/evolution/
- Replace promoteRecentReplaysForCycle() with bundleWarmAssetsForCycle()
- Remove R2 pruning logic from main loop (no longer needed)
- Add unit tests for all bundling functions with mock B2 client

Replays are served gzipped (as-is from B2) to keep deploy size under Pages'
25MB file limit. Frontend will gunzip client-side (separate bead bf-5cwi).

All tests pass (go test ./...).

Closes: bf-3e60
This commit is contained in:
jedarden 2026-05-26 07:35:48 -04:00
parent ad70675c38
commit 4b7d81db45
3 changed files with 415 additions and 32 deletions

View file

@ -16,6 +16,12 @@ import (
"github.com/aicodebattle/acb/metrics"
)
// B2Client defines the interface for B2 operations needed by bundling functions.
// This allows both real S3Client and mock clients to be used.
type B2Client interface {
downloadObject(ctx context.Context, key string) (io.ReadCloser, error)
}
// fetchExemptMatchIDs retrieves match IDs that should never be pruned (from
// series, seasons, and playlists).
func fetchExemptMatchIDs(ctx context.Context, db *sql.DB, outputDir string) (map[string]bool, error) {
@ -428,6 +434,175 @@ func copyWebAssets(cfg *Config, webDistDir string) error {
return nil
}
// bundleWarmReplays copies warm-set replays from B2 into dist/data/replays/
// as gzipped files to be served as static Pages assets.
func bundleWarmReplays(ctx context.Context, cfg *Config, b2Client B2Client, matchIDs []string) error {
if len(matchIDs) == 0 {
return nil
}
// Create output directory
replayDir := filepath.Join(cfg.OutputDir, "data", "replays")
if err := os.MkdirAll(replayDir, 0755); err != nil {
return fmt.Errorf("create replay dir: %w", err)
}
bundled := 0
for _, matchID := range matchIDs {
// Try .json.gz first (standard format)
b2Key := fmt.Sprintf("replays/%s.json.gz", matchID)
body, err := b2Client.downloadObject(ctx, b2Key)
if err != nil {
slog.Warn("Failed to download replay from B2", "match_id", matchID, "error", err)
continue
}
destPath := filepath.Join(replayDir, matchID+".json.gz")
destFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
body.Close()
slog.Error("Failed to create replay file", "match_id", matchID, "error", err)
continue
}
if _, err := io.Copy(destFile, body); err != nil {
destFile.Close()
body.Close()
slog.Error("Failed to write replay file", "match_id", matchID, "error", err)
continue
}
destFile.Close()
body.Close()
bundled++
}
slog.Info("Bundled warm replays", "count", bundled, "total", len(matchIDs))
return nil
}
// bundleWarmThumbnails copies warm-set thumbnails from B2 into dist/data/thumbnails/
func bundleWarmThumbnails(ctx context.Context, cfg *Config, b2Client B2Client, matchIDs []string) error {
if len(matchIDs) == 0 {
return nil
}
// Create output directory
thumbDir := filepath.Join(cfg.OutputDir, "data", "thumbnails")
if err := os.MkdirAll(thumbDir, 0755); err != nil {
return fmt.Errorf("create thumbnail dir: %w", err)
}
bundled := 0
for _, matchID := range matchIDs {
b2Key := fmt.Sprintf("thumbnails/%s.png", matchID)
body, err := b2Client.downloadObject(ctx, b2Key)
if err != nil {
// Thumbnails are optional - don't log as error
continue
}
destPath := filepath.Join(thumbDir, matchID+".png")
destFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
body.Close()
slog.Error("Failed to create thumbnail file", "match_id", matchID, "error", err)
continue
}
if _, err := io.Copy(destFile, body); err != nil {
destFile.Close()
body.Close()
slog.Error("Failed to write thumbnail file", "match_id", matchID, "error", err)
continue
}
destFile.Close()
body.Close()
bundled++
}
slog.Info("Bundled warm thumbnails", "count", bundled)
return nil
}
// bundleWarmCards copies bot profile cards from B2 into dist/data/cards/
func bundleWarmCards(ctx context.Context, cfg *Config, b2Client B2Client, botIDs []string) error {
if len(botIDs) == 0 {
return nil
}
// Create output directory
cardsDir := filepath.Join(cfg.OutputDir, "data", "cards")
if err := os.MkdirAll(cardsDir, 0755); err != nil {
return fmt.Errorf("create cards dir: %w", err)
}
bundled := 0
for _, botID := range botIDs {
b2Key := fmt.Sprintf("cards/%s.png", botID)
body, err := b2Client.downloadObject(ctx, b2Key)
if err != nil {
// Cards are optional - don't log as error
continue
}
destPath := filepath.Join(cardsDir, botID+".png")
destFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
body.Close()
slog.Error("Failed to create card file", "bot_id", botID, "error", err)
continue
}
if _, err := io.Copy(destFile, body); err != nil {
destFile.Close()
body.Close()
slog.Error("Failed to write card file", "bot_id", botID, "error", err)
continue
}
destFile.Close()
body.Close()
bundled++
}
slog.Info("Bundled warm cards", "count", bundled)
return nil
}
// bundleEvolutionLive copies evolution live.json from B2 into dist/data/evolution/
func bundleEvolutionLive(ctx context.Context, cfg *Config, b2Client B2Client) error {
// Create output directory
evolutionDir := filepath.Join(cfg.OutputDir, "data", "evolution")
if err := os.MkdirAll(evolutionDir, 0755); err != nil {
return fmt.Errorf("create evolution dir: %w", err)
}
b2Key := "evolution/live.json"
body, err := b2Client.downloadObject(ctx, b2Key)
if err != nil {
slog.Debug("No evolution live.json in B2", "error", err)
return nil // Not an error - live.json may not exist yet
}
defer body.Close()
destPath := filepath.Join(evolutionDir, "live.json")
destFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return fmt.Errorf("create live.json file: %w", err)
}
defer destFile.Close()
if _, err := io.Copy(destFile, body); err != nil {
return fmt.Errorf("write live.json: %w", err)
}
slog.Info("Bundled evolution live.json")
return nil
}
// writeBuildManifest writes a manifest.json with build metadata
func writeBuildManifest(cfg *Config, buildTime time.Time) error {
manifest := map[string]interface{}{

View file

@ -108,15 +108,6 @@ func main() {
slog.Info("Deployed to Cloudflare Pages")
}
// Run R2 pruning once per week (Monday)
if time.Now().Weekday() == time.Monday {
slog.Info("Running weekly R2 pruning")
if err := pruneR2Cache(context.Background(), cfg); err != nil {
slog.Error("R2 pruning failed", "error", err)
} else {
slog.Info("R2 pruning completed")
}
}
}
// Sleep until next cycle
@ -243,21 +234,15 @@ func runBuildCycle(ctx context.Context, db *sql.DB, cfg *Config) error {
// Non-fatal - continue with rest of build
}
// Upload cards to R2 warm cache
if err := uploadCardsToR2(ctx, cfg, cfg.OutputDir); err != nil {
slog.Error("Failed to upload cards to R2", "error", err)
// Non-fatal
}
// Upload cards to B2 cold archive
// Upload cards to B2 cold archive (primary storage)
if err := uploadCardsToB2(ctx, cfg, cfg.OutputDir); err != nil {
slog.Error("Failed to upload cards to B2", "error", err)
// Non-fatal
}
// Promote recent replays from B2 to R2 warm cache
if err := promoteRecentReplaysForCycle(ctx, db, cfg); err != nil {
slog.Error("Failed to promote recent replays", "error", err)
// Bundle warm-set assets (replays, thumbnails, cards, evolution) from B2 into Pages deploy
if err := bundleWarmAssetsForCycle(ctx, db, cfg, data); err != nil {
slog.Error("Failed to bundle warm assets", "error", err)
// Non-fatal
}
@ -276,8 +261,9 @@ func runBuildCycle(ctx context.Context, db *sql.DB, cfg *Config) error {
return nil
}
// promoteRecentReplaysForCycle promotes recent replays from B2 to R2
func promoteRecentReplaysForCycle(ctx context.Context, db *sql.DB, cfg *Config) error {
// bundleWarmAssetsForCycle bundles warm-set assets from B2 into the Pages deploy directory.
// This includes replays, thumbnails, bot cards, and evolution live.json.
func bundleWarmAssetsForCycle(ctx context.Context, db *sql.DB, cfg *Config, data *IndexData) error {
// Get recent match IDs from the last 24 hours
recentMatchIDs, err := fetchRecentMatchIDs(ctx, db, 24*time.Hour)
if err != nil {
@ -285,34 +271,80 @@ func promoteRecentReplaysForCycle(ctx context.Context, db *sql.DB, cfg *Config)
}
if len(recentMatchIDs) == 0 {
slog.Debug("No recent matches to promote")
slog.Debug("No recent matches to bundle")
return nil
}
// Get exempt match IDs (playlists, series, seasons)
// Get exempt match IDs (playlists, series, seasons) - these are also part of warm set
exemptMatchIDs, err := fetchExemptMatchIDs(ctx, db, cfg.OutputDir)
if err != nil {
slog.Warn("Failed to fetch exempt match IDs, promoting all", "error", err)
slog.Warn("Failed to fetch exempt match IDs, bundling only recent", "error", err)
exemptMatchIDs = make(map[string]bool)
}
// Combine recent and exempt matches for promotion
matchIDsToPromote := recentMatchIDs
// Combine recent and exempt matches for bundling
matchIDsToBundle := recentMatchIDs
for matchID := range exemptMatchIDs {
matchIDsToPromote = append(matchIDsToPromote, matchID)
matchIDsToBundle = append(matchIDsToBundle, matchID)
}
if len(matchIDsToPromote) == 0 {
slog.Debug("No matches to promote")
// Deduplicate match IDs
uniqueMatchIDs := make(map[string]bool)
for _, id := range matchIDsToBundle {
uniqueMatchIDs[id] = true
}
var dedupedMatchIDs []string
for id := range uniqueMatchIDs {
dedupedMatchIDs = append(dedupedMatchIDs, id)
}
if len(dedupedMatchIDs) == 0 {
slog.Debug("No matches to bundle")
return nil
}
slog.Info("Promoting replays to R2",
slog.Info("Bundling warm assets from B2",
"recent_count", len(recentMatchIDs),
"exempt_count", len(exemptMatchIDs),
"total", len(matchIDsToPromote))
"total_unique", len(dedupedMatchIDs))
return promoteRecentReplays(ctx, cfg, matchIDsToPromote)
// Create B2 client for bundling
b2Client, err := getB2Client(cfg)
if err != nil {
return fmt.Errorf("create B2 client: %w", err)
}
// Bundle replays
if err := bundleWarmReplays(ctx, cfg, b2Client, dedupedMatchIDs); err != nil {
slog.Error("Failed to bundle warm replays", "error", err)
// Continue with other assets
}
// Bundle thumbnails
if err := bundleWarmThumbnails(ctx, cfg, b2Client, dedupedMatchIDs); err != nil {
slog.Error("Failed to bundle warm thumbnails", "error", err)
// Continue
}
// Bundle bot cards for all active bots
var botIDs []string
for _, bot := range data.Bots {
botIDs = append(botIDs, bot.ID)
}
if len(botIDs) > 0 {
if err := bundleWarmCards(ctx, cfg, b2Client, botIDs); err != nil {
slog.Error("Failed to bundle warm cards", "error", err)
// Continue
}
}
// Bundle evolution live.json
if err := bundleEvolutionLive(ctx, cfg, b2Client); err != nil {
slog.Error("Failed to bundle evolution live.json", "error", err)
// Continue
}
return nil
}
// fetchRecentMatchIDs retrieves match IDs from the last duration

View file

@ -4,6 +4,8 @@ import (
"bytes"
"context"
"io"
"os"
"path/filepath"
"sort"
"testing"
"time"
@ -253,3 +255,177 @@ func TestMockS3ClientList(t *testing.T) {
t.Error("expected objects sorted oldest first")
}
}
// Test bundleWarmReplays with mock B2 client
func TestBundleWarmReplays(t *testing.T) {
ctx := context.Background()
mockClient := NewMockS3Client()
// Add mock replays to B2
mockClient.Objects["replays/match1.json.gz"] = MockObject{
Content: []byte(`{"turn": 1, "events": []}`),
LastModified: time.Now(),
}
mockClient.Objects["replays/match2.json.gz"] = MockObject{
Content: []byte(`{"turn": 1, "events": []}`),
LastModified: time.Now(),
}
// Create temporary output directory
tmpDir, err := os.MkdirTemp("", "bundle-test-")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
cfg := &Config{OutputDir: tmpDir}
matchIDs := []string{"match1", "match2"}
// Bundle replays
err = bundleWarmReplays(ctx, cfg, mockClient, matchIDs)
if err != nil {
t.Fatalf("bundleWarmReplays failed: %v", err)
}
// Verify files were created
for _, matchID := range matchIDs {
expectedPath := filepath.Join(tmpDir, "data", "replays", matchID+".json.gz")
if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
t.Errorf("expected replay file not created: %s", expectedPath)
}
}
}
// Test bundleWarmThumbnails with mock B2 client
func TestBundleWarmThumbnails(t *testing.T) {
ctx := context.Background()
mockClient := NewMockS3Client()
// Add mock thumbnails to B2
mockClient.Objects["thumbnails/match1.png"] = MockObject{
Content: []byte("fake-png-data"),
LastModified: time.Now(),
}
// Create temporary output directory
tmpDir, err := os.MkdirTemp("", "bundle-test-")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
cfg := &Config{OutputDir: tmpDir}
matchIDs := []string{"match1"}
// Bundle thumbnails
err = bundleWarmThumbnails(ctx, cfg, mockClient, matchIDs)
if err != nil {
t.Fatalf("bundleWarmThumbnails failed: %v", err)
}
// Verify file was created
expectedPath := filepath.Join(tmpDir, "data", "thumbnails", "match1.png")
if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
t.Errorf("expected thumbnail file not created: %s", expectedPath)
}
}
// Test bundleWarmCards with mock B2 client
func TestBundleWarmCards(t *testing.T) {
ctx := context.Background()
mockClient := NewMockS3Client()
// Add mock cards to B2
mockClient.Objects["cards/bot1.png"] = MockObject{
Content: []byte("fake-png-data"),
LastModified: time.Now(),
}
// Create temporary output directory
tmpDir, err := os.MkdirTemp("", "bundle-test-")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
cfg := &Config{OutputDir: tmpDir}
botIDs := []string{"bot1"}
// Bundle cards
err = bundleWarmCards(ctx, cfg, mockClient, botIDs)
if err != nil {
t.Fatalf("bundleWarmCards failed: %v", err)
}
// Verify file was created
expectedPath := filepath.Join(tmpDir, "data", "cards", "bot1.png")
if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
t.Errorf("expected card file not created: %s", expectedPath)
}
}
// Test bundleEvolutionLive with mock B2 client
func TestBundleEvolutionLive(t *testing.T) {
ctx := context.Background()
mockClient := NewMockS3Client()
// Add mock live.json to B2
liveData := `{"updated_at": "2026-05-26T00:00:00Z", "lineage": []}`
mockClient.Objects["evolution/live.json"] = MockObject{
Content: []byte(liveData),
LastModified: time.Now(),
}
// Create temporary output directory
tmpDir, err := os.MkdirTemp("", "bundle-test-")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
cfg := &Config{OutputDir: tmpDir}
// Bundle evolution live.json
err = bundleEvolutionLive(ctx, cfg, mockClient)
if err != nil {
t.Fatalf("bundleEvolutionLive failed: %v", err)
}
// Verify file was created
expectedPath := filepath.Join(tmpDir, "data", "evolution", "live.json")
if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
t.Errorf("expected live.json file not created: %s", expectedPath)
}
// Verify content
content, err := os.ReadFile(expectedPath)
if err != nil {
t.Fatalf("failed to read live.json: %v", err)
}
if string(content) != liveData {
t.Errorf("live.json content mismatch, got %s", string(content))
}
}
// Test bundleWarmReplays with missing B2 objects (graceful handling)
func TestBundleWarmReplaysMissingObjects(t *testing.T) {
ctx := context.Background()
mockClient := NewMockS3Client() // Empty - no objects
// Create temporary output directory
tmpDir, err := os.MkdirTemp("", "bundle-test-")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
cfg := &Config{OutputDir: tmpDir}
matchIDs := []string{"nonexistent"}
// Should not error when objects are missing
err = bundleWarmReplays(ctx, cfg, mockClient, matchIDs)
if err != nil {
t.Errorf("bundleWarmReplays should not error on missing objects, got: %v", err)
}
}