From 4b7d81db45a0fbd5eb6b2f63b8278ca910acd983 Mon Sep 17 00:00:00 2001 From: jedarden Date: Tue, 26 May 2026 07:35:48 -0400 Subject: [PATCH] feat(index-builder): bundle warm-set replays as static Pages assets Per bead bf-3e60: instead of copying B2->R2 (warm cache), bundle warm-set replays, thumbnails, cards, and evolution live.json directly into the Pages deploy directory as static assets (dist/data/). This serves replays same-origin, eliminating R2 dependency and 404 errors. Changes: - Add B2Client interface for testable B2 operations - Add bundleWarmReplays(): copies replays/*.json.gz from B2 to dist/data/replays/ - Add bundleWarmThumbnails(): copies thumbnails/*.png from B2 to dist/data/thumbnails/ - Add bundleWarmCards(): copies cards/*.png from B2 to dist/data/cards/ - Add bundleEvolutionLive(): copies evolution/live.json from B2 to dist/data/evolution/ - Replace promoteRecentReplaysForCycle() with bundleWarmAssetsForCycle() - Remove R2 pruning logic from main loop (no longer needed) - Add unit tests for all bundling functions with mock B2 client Replays are served gzipped (as-is from B2) to keep deploy size under Pages' 25MB file limit. Frontend will gunzip client-side (separate bead bf-5cwi). All tests pass (go test ./...). Closes: bf-3e60 --- cmd/acb-index-builder/deploy.go | 175 ++++++++++++++++++++++++++++++ cmd/acb-index-builder/main.go | 96 +++++++++++------ cmd/acb-index-builder/s3_test.go | 176 +++++++++++++++++++++++++++++++ 3 files changed, 415 insertions(+), 32 deletions(-) diff --git a/cmd/acb-index-builder/deploy.go b/cmd/acb-index-builder/deploy.go index 5f353dd..b959cc1 100644 --- a/cmd/acb-index-builder/deploy.go +++ b/cmd/acb-index-builder/deploy.go @@ -16,6 +16,12 @@ import ( "github.com/aicodebattle/acb/metrics" ) +// B2Client defines the interface for B2 operations needed by bundling functions. +// This allows both real S3Client and mock clients to be used. +type B2Client interface { + downloadObject(ctx context.Context, key string) (io.ReadCloser, error) +} + // fetchExemptMatchIDs retrieves match IDs that should never be pruned (from // series, seasons, and playlists). func fetchExemptMatchIDs(ctx context.Context, db *sql.DB, outputDir string) (map[string]bool, error) { @@ -428,6 +434,175 @@ func copyWebAssets(cfg *Config, webDistDir string) error { return nil } +// bundleWarmReplays copies warm-set replays from B2 into dist/data/replays/ +// as gzipped files to be served as static Pages assets. +func bundleWarmReplays(ctx context.Context, cfg *Config, b2Client B2Client, matchIDs []string) error { + if len(matchIDs) == 0 { + return nil + } + + // Create output directory + replayDir := filepath.Join(cfg.OutputDir, "data", "replays") + if err := os.MkdirAll(replayDir, 0755); err != nil { + return fmt.Errorf("create replay dir: %w", err) + } + + bundled := 0 + for _, matchID := range matchIDs { + // Try .json.gz first (standard format) + b2Key := fmt.Sprintf("replays/%s.json.gz", matchID) + body, err := b2Client.downloadObject(ctx, b2Key) + if err != nil { + slog.Warn("Failed to download replay from B2", "match_id", matchID, "error", err) + continue + } + + destPath := filepath.Join(replayDir, matchID+".json.gz") + destFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + body.Close() + slog.Error("Failed to create replay file", "match_id", matchID, "error", err) + continue + } + + if _, err := io.Copy(destFile, body); err != nil { + destFile.Close() + body.Close() + slog.Error("Failed to write replay file", "match_id", matchID, "error", err) + continue + } + + destFile.Close() + body.Close() + bundled++ + } + + slog.Info("Bundled warm replays", "count", bundled, "total", len(matchIDs)) + return nil +} + +// bundleWarmThumbnails copies warm-set thumbnails from B2 into dist/data/thumbnails/ +func bundleWarmThumbnails(ctx context.Context, cfg *Config, b2Client B2Client, matchIDs []string) error { + if len(matchIDs) == 0 { + return nil + } + + // Create output directory + thumbDir := filepath.Join(cfg.OutputDir, "data", "thumbnails") + if err := os.MkdirAll(thumbDir, 0755); err != nil { + return fmt.Errorf("create thumbnail dir: %w", err) + } + + bundled := 0 + for _, matchID := range matchIDs { + b2Key := fmt.Sprintf("thumbnails/%s.png", matchID) + body, err := b2Client.downloadObject(ctx, b2Key) + if err != nil { + // Thumbnails are optional - don't log as error + continue + } + + destPath := filepath.Join(thumbDir, matchID+".png") + destFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + body.Close() + slog.Error("Failed to create thumbnail file", "match_id", matchID, "error", err) + continue + } + + if _, err := io.Copy(destFile, body); err != nil { + destFile.Close() + body.Close() + slog.Error("Failed to write thumbnail file", "match_id", matchID, "error", err) + continue + } + + destFile.Close() + body.Close() + bundled++ + } + + slog.Info("Bundled warm thumbnails", "count", bundled) + return nil +} + +// bundleWarmCards copies bot profile cards from B2 into dist/data/cards/ +func bundleWarmCards(ctx context.Context, cfg *Config, b2Client B2Client, botIDs []string) error { + if len(botIDs) == 0 { + return nil + } + + // Create output directory + cardsDir := filepath.Join(cfg.OutputDir, "data", "cards") + if err := os.MkdirAll(cardsDir, 0755); err != nil { + return fmt.Errorf("create cards dir: %w", err) + } + + bundled := 0 + for _, botID := range botIDs { + b2Key := fmt.Sprintf("cards/%s.png", botID) + body, err := b2Client.downloadObject(ctx, b2Key) + if err != nil { + // Cards are optional - don't log as error + continue + } + + destPath := filepath.Join(cardsDir, botID+".png") + destFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + body.Close() + slog.Error("Failed to create card file", "bot_id", botID, "error", err) + continue + } + + if _, err := io.Copy(destFile, body); err != nil { + destFile.Close() + body.Close() + slog.Error("Failed to write card file", "bot_id", botID, "error", err) + continue + } + + destFile.Close() + body.Close() + bundled++ + } + + slog.Info("Bundled warm cards", "count", bundled) + return nil +} + +// bundleEvolutionLive copies evolution live.json from B2 into dist/data/evolution/ +func bundleEvolutionLive(ctx context.Context, cfg *Config, b2Client B2Client) error { + + // Create output directory + evolutionDir := filepath.Join(cfg.OutputDir, "data", "evolution") + if err := os.MkdirAll(evolutionDir, 0755); err != nil { + return fmt.Errorf("create evolution dir: %w", err) + } + + b2Key := "evolution/live.json" + body, err := b2Client.downloadObject(ctx, b2Key) + if err != nil { + slog.Debug("No evolution live.json in B2", "error", err) + return nil // Not an error - live.json may not exist yet + } + defer body.Close() + + destPath := filepath.Join(evolutionDir, "live.json") + destFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("create live.json file: %w", err) + } + defer destFile.Close() + + if _, err := io.Copy(destFile, body); err != nil { + return fmt.Errorf("write live.json: %w", err) + } + + slog.Info("Bundled evolution live.json") + return nil +} + // writeBuildManifest writes a manifest.json with build metadata func writeBuildManifest(cfg *Config, buildTime time.Time) error { manifest := map[string]interface{}{ diff --git a/cmd/acb-index-builder/main.go b/cmd/acb-index-builder/main.go index b322505..944d1e4 100644 --- a/cmd/acb-index-builder/main.go +++ b/cmd/acb-index-builder/main.go @@ -108,15 +108,6 @@ func main() { slog.Info("Deployed to Cloudflare Pages") } - // Run R2 pruning once per week (Monday) - if time.Now().Weekday() == time.Monday { - slog.Info("Running weekly R2 pruning") - if err := pruneR2Cache(context.Background(), cfg); err != nil { - slog.Error("R2 pruning failed", "error", err) - } else { - slog.Info("R2 pruning completed") - } - } } // Sleep until next cycle @@ -243,21 +234,15 @@ func runBuildCycle(ctx context.Context, db *sql.DB, cfg *Config) error { // Non-fatal - continue with rest of build } - // Upload cards to R2 warm cache - if err := uploadCardsToR2(ctx, cfg, cfg.OutputDir); err != nil { - slog.Error("Failed to upload cards to R2", "error", err) - // Non-fatal - } - - // Upload cards to B2 cold archive + // Upload cards to B2 cold archive (primary storage) if err := uploadCardsToB2(ctx, cfg, cfg.OutputDir); err != nil { slog.Error("Failed to upload cards to B2", "error", err) // Non-fatal } - // Promote recent replays from B2 to R2 warm cache - if err := promoteRecentReplaysForCycle(ctx, db, cfg); err != nil { - slog.Error("Failed to promote recent replays", "error", err) + // Bundle warm-set assets (replays, thumbnails, cards, evolution) from B2 into Pages deploy + if err := bundleWarmAssetsForCycle(ctx, db, cfg, data); err != nil { + slog.Error("Failed to bundle warm assets", "error", err) // Non-fatal } @@ -276,8 +261,9 @@ func runBuildCycle(ctx context.Context, db *sql.DB, cfg *Config) error { return nil } -// promoteRecentReplaysForCycle promotes recent replays from B2 to R2 -func promoteRecentReplaysForCycle(ctx context.Context, db *sql.DB, cfg *Config) error { +// bundleWarmAssetsForCycle bundles warm-set assets from B2 into the Pages deploy directory. +// This includes replays, thumbnails, bot cards, and evolution live.json. +func bundleWarmAssetsForCycle(ctx context.Context, db *sql.DB, cfg *Config, data *IndexData) error { // Get recent match IDs from the last 24 hours recentMatchIDs, err := fetchRecentMatchIDs(ctx, db, 24*time.Hour) if err != nil { @@ -285,34 +271,80 @@ func promoteRecentReplaysForCycle(ctx context.Context, db *sql.DB, cfg *Config) } if len(recentMatchIDs) == 0 { - slog.Debug("No recent matches to promote") + slog.Debug("No recent matches to bundle") return nil } - // Get exempt match IDs (playlists, series, seasons) + // Get exempt match IDs (playlists, series, seasons) - these are also part of warm set exemptMatchIDs, err := fetchExemptMatchIDs(ctx, db, cfg.OutputDir) if err != nil { - slog.Warn("Failed to fetch exempt match IDs, promoting all", "error", err) + slog.Warn("Failed to fetch exempt match IDs, bundling only recent", "error", err) exemptMatchIDs = make(map[string]bool) } - // Combine recent and exempt matches for promotion - matchIDsToPromote := recentMatchIDs + // Combine recent and exempt matches for bundling + matchIDsToBundle := recentMatchIDs for matchID := range exemptMatchIDs { - matchIDsToPromote = append(matchIDsToPromote, matchID) + matchIDsToBundle = append(matchIDsToBundle, matchID) } - if len(matchIDsToPromote) == 0 { - slog.Debug("No matches to promote") + // Deduplicate match IDs + uniqueMatchIDs := make(map[string]bool) + for _, id := range matchIDsToBundle { + uniqueMatchIDs[id] = true + } + var dedupedMatchIDs []string + for id := range uniqueMatchIDs { + dedupedMatchIDs = append(dedupedMatchIDs, id) + } + + if len(dedupedMatchIDs) == 0 { + slog.Debug("No matches to bundle") return nil } - slog.Info("Promoting replays to R2", + slog.Info("Bundling warm assets from B2", "recent_count", len(recentMatchIDs), "exempt_count", len(exemptMatchIDs), - "total", len(matchIDsToPromote)) + "total_unique", len(dedupedMatchIDs)) - return promoteRecentReplays(ctx, cfg, matchIDsToPromote) + // Create B2 client for bundling + b2Client, err := getB2Client(cfg) + if err != nil { + return fmt.Errorf("create B2 client: %w", err) + } + + // Bundle replays + if err := bundleWarmReplays(ctx, cfg, b2Client, dedupedMatchIDs); err != nil { + slog.Error("Failed to bundle warm replays", "error", err) + // Continue with other assets + } + + // Bundle thumbnails + if err := bundleWarmThumbnails(ctx, cfg, b2Client, dedupedMatchIDs); err != nil { + slog.Error("Failed to bundle warm thumbnails", "error", err) + // Continue + } + + // Bundle bot cards for all active bots + var botIDs []string + for _, bot := range data.Bots { + botIDs = append(botIDs, bot.ID) + } + if len(botIDs) > 0 { + if err := bundleWarmCards(ctx, cfg, b2Client, botIDs); err != nil { + slog.Error("Failed to bundle warm cards", "error", err) + // Continue + } + } + + // Bundle evolution live.json + if err := bundleEvolutionLive(ctx, cfg, b2Client); err != nil { + slog.Error("Failed to bundle evolution live.json", "error", err) + // Continue + } + + return nil } // fetchRecentMatchIDs retrieves match IDs from the last duration diff --git a/cmd/acb-index-builder/s3_test.go b/cmd/acb-index-builder/s3_test.go index e82d80a..b43f896 100644 --- a/cmd/acb-index-builder/s3_test.go +++ b/cmd/acb-index-builder/s3_test.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "io" + "os" + "path/filepath" "sort" "testing" "time" @@ -253,3 +255,177 @@ func TestMockS3ClientList(t *testing.T) { t.Error("expected objects sorted oldest first") } } + +// Test bundleWarmReplays with mock B2 client +func TestBundleWarmReplays(t *testing.T) { + ctx := context.Background() + mockClient := NewMockS3Client() + + // Add mock replays to B2 + mockClient.Objects["replays/match1.json.gz"] = MockObject{ + Content: []byte(`{"turn": 1, "events": []}`), + LastModified: time.Now(), + } + mockClient.Objects["replays/match2.json.gz"] = MockObject{ + Content: []byte(`{"turn": 1, "events": []}`), + LastModified: time.Now(), + } + + // Create temporary output directory + tmpDir, err := os.MkdirTemp("", "bundle-test-") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + cfg := &Config{OutputDir: tmpDir} + matchIDs := []string{"match1", "match2"} + + // Bundle replays + err = bundleWarmReplays(ctx, cfg, mockClient, matchIDs) + if err != nil { + t.Fatalf("bundleWarmReplays failed: %v", err) + } + + // Verify files were created + for _, matchID := range matchIDs { + expectedPath := filepath.Join(tmpDir, "data", "replays", matchID+".json.gz") + if _, err := os.Stat(expectedPath); os.IsNotExist(err) { + t.Errorf("expected replay file not created: %s", expectedPath) + } + } +} + +// Test bundleWarmThumbnails with mock B2 client +func TestBundleWarmThumbnails(t *testing.T) { + ctx := context.Background() + mockClient := NewMockS3Client() + + // Add mock thumbnails to B2 + mockClient.Objects["thumbnails/match1.png"] = MockObject{ + Content: []byte("fake-png-data"), + LastModified: time.Now(), + } + + // Create temporary output directory + tmpDir, err := os.MkdirTemp("", "bundle-test-") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + cfg := &Config{OutputDir: tmpDir} + matchIDs := []string{"match1"} + + // Bundle thumbnails + err = bundleWarmThumbnails(ctx, cfg, mockClient, matchIDs) + if err != nil { + t.Fatalf("bundleWarmThumbnails failed: %v", err) + } + + // Verify file was created + expectedPath := filepath.Join(tmpDir, "data", "thumbnails", "match1.png") + if _, err := os.Stat(expectedPath); os.IsNotExist(err) { + t.Errorf("expected thumbnail file not created: %s", expectedPath) + } +} + +// Test bundleWarmCards with mock B2 client +func TestBundleWarmCards(t *testing.T) { + ctx := context.Background() + mockClient := NewMockS3Client() + + // Add mock cards to B2 + mockClient.Objects["cards/bot1.png"] = MockObject{ + Content: []byte("fake-png-data"), + LastModified: time.Now(), + } + + // Create temporary output directory + tmpDir, err := os.MkdirTemp("", "bundle-test-") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + cfg := &Config{OutputDir: tmpDir} + botIDs := []string{"bot1"} + + // Bundle cards + err = bundleWarmCards(ctx, cfg, mockClient, botIDs) + if err != nil { + t.Fatalf("bundleWarmCards failed: %v", err) + } + + // Verify file was created + expectedPath := filepath.Join(tmpDir, "data", "cards", "bot1.png") + if _, err := os.Stat(expectedPath); os.IsNotExist(err) { + t.Errorf("expected card file not created: %s", expectedPath) + } +} + +// Test bundleEvolutionLive with mock B2 client +func TestBundleEvolutionLive(t *testing.T) { + ctx := context.Background() + mockClient := NewMockS3Client() + + // Add mock live.json to B2 + liveData := `{"updated_at": "2026-05-26T00:00:00Z", "lineage": []}` + mockClient.Objects["evolution/live.json"] = MockObject{ + Content: []byte(liveData), + LastModified: time.Now(), + } + + // Create temporary output directory + tmpDir, err := os.MkdirTemp("", "bundle-test-") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + cfg := &Config{OutputDir: tmpDir} + + // Bundle evolution live.json + err = bundleEvolutionLive(ctx, cfg, mockClient) + if err != nil { + t.Fatalf("bundleEvolutionLive failed: %v", err) + } + + // Verify file was created + expectedPath := filepath.Join(tmpDir, "data", "evolution", "live.json") + if _, err := os.Stat(expectedPath); os.IsNotExist(err) { + t.Errorf("expected live.json file not created: %s", expectedPath) + } + + // Verify content + content, err := os.ReadFile(expectedPath) + if err != nil { + t.Fatalf("failed to read live.json: %v", err) + } + if string(content) != liveData { + t.Errorf("live.json content mismatch, got %s", string(content)) + } +} + +// Test bundleWarmReplays with missing B2 objects (graceful handling) +func TestBundleWarmReplaysMissingObjects(t *testing.T) { + ctx := context.Background() + mockClient := NewMockS3Client() // Empty - no objects + + // Create temporary output directory + tmpDir, err := os.MkdirTemp("", "bundle-test-") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + cfg := &Config{OutputDir: tmpDir} + matchIDs := []string{"nonexistent"} + + // Should not error when objects are missing + err = bundleWarmReplays(ctx, cfg, mockClient, matchIDs) + if err != nil { + t.Errorf("bundleWarmReplays should not error on missing objects, got: %v", err) + } +} +