- Add s3.go with AWS SDK v2 S3Client wrapper for R2/B2 operations - Implement listObjects, deleteObject, objectExists, uploadFile, copyObject, downloadObject - Add s3_test.go with MockS3Client and comprehensive tests - Wire promoteRecentReplaysForCycle() into build cycle in main.go - Add fetchRecentMatchIDs() to query recent matches from PostgreSQL - Add fetchExemptMatchIDs() to protect series/season/playlist matches from pruning - Implement pruneR2CacheWithDB() for 10GB cap enforcement with exemptions - Update go.mod with AWS SDK v2 dependencies Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
388 lines
10 KiB
Go
388 lines
10 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// fetchExemptMatchIDs retrieves match IDs that should never be pruned (from series, seasons, playlists)
|
|
func fetchExemptMatchIDs(ctx context.Context, db *sql.DB) (map[string]bool, error) {
|
|
if db == nil {
|
|
return make(map[string]bool), nil
|
|
}
|
|
|
|
exempt := make(map[string]bool)
|
|
|
|
// Matches in active series
|
|
seriesQuery := `
|
|
SELECT DISTINCT sm.match_id
|
|
FROM series_matches sm
|
|
JOIN series s ON sm.series_id = s.id
|
|
WHERE s.status IN ('active', 'pending')
|
|
`
|
|
rows, err := db.QueryContext(ctx, seriesQuery)
|
|
if err == nil {
|
|
for rows.Next() {
|
|
var id string
|
|
if err := rows.Scan(&id); err == nil {
|
|
exempt[id] = true
|
|
}
|
|
}
|
|
rows.Close()
|
|
}
|
|
|
|
// Matches in active seasons
|
|
seasonQuery := `
|
|
SELECT DISTINCT match_id
|
|
FROM season_matches
|
|
WHERE season_id IN (
|
|
SELECT id FROM seasons WHERE ends_at IS NULL OR ends_at > NOW()
|
|
)
|
|
`
|
|
rows, err = db.QueryContext(ctx, seasonQuery)
|
|
if err == nil {
|
|
for rows.Next() {
|
|
var id string
|
|
if err := rows.Scan(&id); err == nil {
|
|
exempt[id] = true
|
|
}
|
|
}
|
|
rows.Close()
|
|
}
|
|
|
|
// Matches in featured playlists
|
|
playlistQuery := `
|
|
SELECT DISTINCT pm.match_id
|
|
FROM playlist_matches pm
|
|
JOIN playlists p ON pm.playlist_id = p.id
|
|
WHERE p.featured = true
|
|
`
|
|
rows, err = db.QueryContext(ctx, playlistQuery)
|
|
if err == nil {
|
|
for rows.Next() {
|
|
var id string
|
|
if err := rows.Scan(&id); err == nil {
|
|
exempt[id] = true
|
|
}
|
|
}
|
|
rows.Close()
|
|
}
|
|
|
|
slog.Debug("Fetched exempt match IDs for pruning", "count", len(exempt))
|
|
return exempt, nil
|
|
}
|
|
|
|
// deployToPages deploys the generated files to Cloudflare Pages via wrangler
|
|
func deployToPages(cfg *Config) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
|
|
// Check if wrangler is available
|
|
if _, err := exec.LookPath("wrangler"); err != nil {
|
|
return fmt.Errorf("wrangler not found in PATH: %w", err)
|
|
}
|
|
|
|
// Set up environment for wrangler
|
|
env := os.Environ()
|
|
if cfg.CloudflareAPIToken != "" {
|
|
env = append(env, fmt.Sprintf("CLOUDFLARE_API_TOKEN=%s", cfg.CloudflareAPIToken))
|
|
}
|
|
if cfg.CloudflareAccountID != "" {
|
|
env = append(env, fmt.Sprintf("CLOUDFLARE_ACCOUNT_ID=%s", cfg.CloudflareAccountID))
|
|
}
|
|
|
|
// Run wrangler pages deploy
|
|
args := []string{
|
|
"pages", "deploy",
|
|
cfg.OutputDir,
|
|
"--project-name", cfg.PagesProjectName,
|
|
"--branch", "main",
|
|
}
|
|
|
|
cmd := exec.CommandContext(ctx, "wrangler", args...)
|
|
cmd.Env = env
|
|
cmd.Stdout = os.Stdout
|
|
cmd.Stderr = os.Stderr
|
|
|
|
slog.Info("Running wrangler pages deploy",
|
|
"project", cfg.PagesProjectName,
|
|
"directory", cfg.OutputDir,
|
|
)
|
|
|
|
if err := cmd.Run(); err != nil {
|
|
return fmt.Errorf("wrangler pages deploy failed: %w", err)
|
|
}
|
|
|
|
slog.Info("Successfully deployed to Cloudflare Pages")
|
|
return nil
|
|
}
|
|
|
|
// pruneR2Cache removes old replays from R2 warm cache to stay within the 10GB free tier
|
|
// It also promotes recent replays from B2 to R2
|
|
func pruneR2Cache(ctx context.Context, cfg *Config) error {
|
|
return pruneR2CacheWithDB(ctx, cfg, nil)
|
|
}
|
|
|
|
// pruneR2CacheWithDB removes old replays from R2, respecting exempt matches
|
|
func pruneR2CacheWithDB(ctx context.Context, cfg *Config, db *sql.DB) error {
|
|
// R2 max size in bytes (10 GB with 500MB buffer for safety)
|
|
maxSize := int64(10*1024*1024*1024 - 500*1024*1024)
|
|
|
|
// List all objects in R2 replays directory
|
|
objects, err := listR2Objects(ctx, cfg, "replays/")
|
|
if err != nil {
|
|
return fmt.Errorf("list R2 objects: %w", err)
|
|
}
|
|
|
|
// Calculate total size
|
|
var totalSize int64
|
|
for _, obj := range objects {
|
|
totalSize += obj.Size
|
|
}
|
|
|
|
slog.Info("R2 warm cache status",
|
|
"objects", len(objects),
|
|
"total_size_gb", float64(totalSize)/(1024*1024*1024),
|
|
"max_size_gb", float64(maxSize)/(1024*1024*1024),
|
|
)
|
|
|
|
// If under limit, nothing to prune
|
|
if totalSize <= maxSize {
|
|
slog.Info("R2 cache within limits, no pruning needed")
|
|
return nil
|
|
}
|
|
|
|
// Get exempt match IDs if db is provided
|
|
exemptMatchIDs := make(map[string]bool)
|
|
if db != nil {
|
|
exemptMatchIDs, err = fetchExemptMatchIDs(ctx, db)
|
|
if err != nil {
|
|
slog.Warn("Failed to fetch exempt match IDs, will proceed without exemptions", "error", err)
|
|
}
|
|
}
|
|
|
|
// Sort objects by age (oldest first) and delete until under limit
|
|
// Objects are already sorted by LastModified from listR2Objects
|
|
toDelete := int64(0)
|
|
prunedCount := 0
|
|
for _, obj := range objects {
|
|
if totalSize-toDelete <= maxSize {
|
|
break
|
|
}
|
|
|
|
// Extract match ID from key (replays/{match_id}.json.gz)
|
|
matchID := extractMatchIDFromKey(obj.Key)
|
|
if exemptMatchIDs[matchID] {
|
|
slog.Debug("Skipping exempt match from pruning", "key", obj.Key, "match_id", matchID)
|
|
continue
|
|
}
|
|
|
|
if err := deleteR2Object(ctx, cfg, obj.Key); err != nil {
|
|
slog.Error("Failed to delete R2 object", "key", obj.Key, "error", err)
|
|
continue
|
|
}
|
|
|
|
toDelete += obj.Size
|
|
prunedCount++
|
|
slog.Info("Pruned R2 object", "key", obj.Key, "size_mb", obj.Size/(1024*1024))
|
|
}
|
|
|
|
slog.Info("R2 pruning complete",
|
|
"pruned_count", prunedCount,
|
|
"pruned_size_gb", float64(toDelete)/(1024*1024*1024),
|
|
)
|
|
|
|
return nil
|
|
}
|
|
|
|
// extractMatchIDFromKey extracts the match ID from a replay key
|
|
func extractMatchIDFromKey(key string) string {
|
|
// Key format: replays/{match_id}.json.gz
|
|
parts := strings.Split(key, "/")
|
|
if len(parts) < 2 {
|
|
return ""
|
|
}
|
|
filename := parts[len(parts)-1]
|
|
// Remove .json.gz extension
|
|
if strings.HasSuffix(filename, ".json.gz") {
|
|
return filename[:len(filename)-8]
|
|
}
|
|
return filename
|
|
}
|
|
|
|
// promoteRecentReplays copies recent replays from B2 to R2 warm cache
|
|
func promoteRecentReplays(ctx context.Context, cfg *Config, matchIDs []string) error {
|
|
for _, matchID := range matchIDs {
|
|
// Source path in B2
|
|
b2Key := fmt.Sprintf("replays/%s.json.gz", matchID)
|
|
|
|
// Check if already in R2
|
|
r2Key := b2Key
|
|
exists, err := checkR2ObjectExists(ctx, cfg, r2Key)
|
|
if err != nil {
|
|
slog.Error("Failed to check R2 object existence", "key", r2Key, "error", err)
|
|
continue
|
|
}
|
|
if exists {
|
|
continue // Already in warm cache
|
|
}
|
|
|
|
// Copy from B2 to R2
|
|
if err := copyB2ToR2(ctx, cfg, b2Key, r2Key); err != nil {
|
|
slog.Error("Failed to promote replay to R2", "match_id", matchID, "error", err)
|
|
continue
|
|
}
|
|
|
|
slog.Info("Promoted replay to R2 warm cache", "match_id", matchID)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// R2Object represents an object in R2 storage
|
|
type R2Object struct {
|
|
Key string
|
|
Size int64
|
|
LastModified time.Time
|
|
}
|
|
|
|
// getR2Client returns an S3 client for R2
|
|
func getR2Client(cfg *Config) (*S3Client, error) {
|
|
if cfg.R2AccessKey == "" || cfg.R2SecretKey == "" || cfg.R2BucketName == "" {
|
|
return nil, fmt.Errorf("R2 credentials not configured")
|
|
}
|
|
return NewS3Client(cfg.R2Endpoint, cfg.R2AccessKey, cfg.R2SecretKey, cfg.R2BucketName)
|
|
}
|
|
|
|
// getB2Client returns an S3 client for B2
|
|
func getB2Client(cfg *Config) (*S3Client, error) {
|
|
if cfg.B2AccessKey == "" || cfg.B2SecretKey == "" || cfg.B2BucketName == "" {
|
|
return nil, fmt.Errorf("B2 credentials not configured")
|
|
}
|
|
return NewS3Client(cfg.B2Endpoint, cfg.B2AccessKey, cfg.B2SecretKey, cfg.B2BucketName)
|
|
}
|
|
|
|
// listR2Objects lists all objects in R2 under a prefix, sorted by LastModified (oldest first)
|
|
func listR2Objects(ctx context.Context, cfg *Config, prefix string) ([]R2Object, error) {
|
|
client, err := getR2Client(cfg)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("create R2 client: %w", err)
|
|
}
|
|
|
|
return client.listObjects(ctx, prefix)
|
|
}
|
|
|
|
// deleteR2Object deletes an object from R2
|
|
func deleteR2Object(ctx context.Context, cfg *Config, key string) error {
|
|
client, err := getR2Client(cfg)
|
|
if err != nil {
|
|
return fmt.Errorf("create R2 client: %w", err)
|
|
}
|
|
|
|
return client.deleteObject(ctx, key)
|
|
}
|
|
|
|
// checkR2ObjectExists checks if an object exists in R2
|
|
func checkR2ObjectExists(ctx context.Context, cfg *Config, key string) (bool, error) {
|
|
client, err := getR2Client(cfg)
|
|
if err != nil {
|
|
return false, fmt.Errorf("create R2 client: %w", err)
|
|
}
|
|
|
|
return client.objectExists(ctx, key)
|
|
}
|
|
|
|
// copyB2ToR2 copies an object from B2 to R2 by downloading from B2 and uploading to R2
|
|
func copyB2ToR2(ctx context.Context, cfg *Config, b2Key, r2Key string) error {
|
|
b2Client, err := getB2Client(cfg)
|
|
if err != nil {
|
|
return fmt.Errorf("create B2 client: %w", err)
|
|
}
|
|
|
|
r2Client, err := getR2Client(cfg)
|
|
if err != nil {
|
|
return fmt.Errorf("create R2 client: %w", err)
|
|
}
|
|
|
|
// Download from B2
|
|
body, err := b2Client.downloadObject(ctx, b2Key)
|
|
if err != nil {
|
|
return fmt.Errorf("download from B2: %w", err)
|
|
}
|
|
defer body.Close()
|
|
|
|
// Upload to R2
|
|
contentType := getS3ContentType(r2Key)
|
|
if err := r2Client.uploadFile(ctx, r2Key, body, contentType); err != nil {
|
|
return fmt.Errorf("upload to R2: %w", err)
|
|
}
|
|
|
|
slog.Info("Copied object from B2 to R2", "b2_key", b2Key, "r2_key", r2Key)
|
|
return nil
|
|
}
|
|
|
|
// copyWebAssets copies the built web SPA to the output directory
|
|
func copyWebAssets(cfg *Config, webDistDir string) error {
|
|
// Copy all files from web/dist to output directory
|
|
err := filepath.Walk(webDistDir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
relPath, err := filepath.Rel(webDistDir, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
destPath := filepath.Join(cfg.OutputDir, relPath)
|
|
|
|
if info.IsDir() {
|
|
return os.MkdirAll(destPath, 0755)
|
|
}
|
|
|
|
// Read source file
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Write to destination
|
|
return os.WriteFile(destPath, data, 0644)
|
|
})
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("copy web assets: %w", err)
|
|
}
|
|
|
|
slog.Info("Copied web assets to output directory", "source", webDistDir)
|
|
return nil
|
|
}
|
|
|
|
// writeBuildManifest writes a manifest.json with build metadata
|
|
func writeBuildManifest(cfg *Config, buildTime time.Time) error {
|
|
manifest := map[string]interface{}{
|
|
"built_at": buildTime.UTC().Format(time.RFC3339),
|
|
"version": "1.0.0",
|
|
"environment": getEnvOrDefault("ACB_ENV", "production"),
|
|
}
|
|
|
|
manifestPath := filepath.Join(cfg.OutputDir, "data", "manifest.json")
|
|
return writeJSON(manifestPath, manifest)
|
|
}
|
|
|
|
func getEnvOrDefault(key, defaultValue string) string {
|
|
if val := os.Getenv(key); val != "" {
|
|
return val
|
|
}
|
|
return defaultValue
|
|
}
|
|
|
|
// ensure valid function references
|
|
var _ = strings.Join
|