From 1523c52e0a5498208a64ec61cc934e7fd45d4480 Mon Sep 17 00:00:00 2001 From: jedarden Date: Sun, 29 Mar 2026 04:53:35 -0400 Subject: [PATCH] Add R2 upload for live evolution observatory (Phase 10) - Add R2 client module (cmd/acb-evolver/internal/live/r2.go) with S3-compatible uploads to Cloudflare R2 - UploadLiveJSON() uploads evolution state to evolution/live.json with Cache-Control: max-age=10 for near-real-time updates - Add -r2 and -r2-only flags to live-export subcommand - Add tests for R2 config validation and credential handling - Update frontend to fetch live data from R2 URL instead of Pages Co-Authored-By: Claude Opus 4.6 --- PROGRESS.md | 17 ++- cmd/acb-evolver/internal/live/exporter.go | 5 + cmd/acb-evolver/internal/live/r2.go | 125 ++++++++++++++++++++++ cmd/acb-evolver/internal/live/r2_test.go | 116 ++++++++++++++++++++ cmd/acb-evolver/main.go | 32 +++++- web/src/api-types.ts | 7 +- 6 files changed, 294 insertions(+), 8 deletions(-) create mode 100644 cmd/acb-evolver/internal/live/r2.go create mode 100644 cmd/acb-evolver/internal/live/r2_test.go diff --git a/PROGRESS.md b/PROGRESS.md index 1c7165e..6cae1df 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -4,9 +4,17 @@ **Status: 🔄 In Progress** -**Last Updated: 2026-03-29** (Blog infrastructure) +**Last Updated: 2026-03-29** (Live evolution observatory) ### Recent Changes (2026-03-29) +- **Phase 10 Live Evolution Observatory** (`cmd/acb-evolver/internal/live/r2.go`): + - R2 client for S3-compatible uploads to Cloudflare R2 + - `UploadLiveJSON()` uploads evolution state to `evolution/live.json` + - Cache-Control: max-age=10 for near-real-time updates (10s polling) + - `live-export -r2` flag enables R2 upload alongside local file + - `live-export -r2-only` flag for R2-only mode (no local file) + - Tests for config validation and credential handling + - Frontend updated to fetch from R2 URL (`https://r2.aicodebattle.com/evolution/live.json`) - **Phase 10 Blog Infrastructure** (`cmd/acb-index-builder/blog.go`, `web/src/pages/blog.ts`): - Weekly meta report generation: auto-generated blog posts with competitive analysis - Story arc chronicles: rise stories, upset narratives, rivalry updates @@ -421,7 +429,12 @@ - Individual post page with markdown rendering - Blog routes added to SPA router - Blog link added to navigation -- [ ] Live evolution observatory (evolver writes live.json to R2) +- [x] Live evolution observatory (evolver writes live.json to R2) + - R2 client module (`cmd/acb-evolver/internal/live/r2.go`) for S3-compatible uploads + - `live-export -r2` and `live-export -r2-only` flags for R2 upload + - Frontend fetches from R2 (`https://r2.aicodebattle.com/evolution/live.json`) + - Cache-Control: max-age=10 for near-real-time updates + - Tests for R2 config validation and credential handling - [ ] Narrative engine (weekly story arc detection + LLM chronicles) - [ ] Public match data documentation (OpenAPI-style) diff --git a/cmd/acb-evolver/internal/live/exporter.go b/cmd/acb-evolver/internal/live/exporter.go index afcfa53..cbb4710 100644 --- a/cmd/acb-evolver/internal/live/exporter.go +++ b/cmd/acb-evolver/internal/live/exporter.go @@ -265,3 +265,8 @@ func dirOf(p string) string { func round3(v float64) float64 { return math.Round(v*1000) / 1000 } + +// marshal returns indented JSON for the live data. +func marshal(d *LiveData) ([]byte, error) { + return json.MarshalIndent(d, "", " ") +} diff --git a/cmd/acb-evolver/internal/live/r2.go b/cmd/acb-evolver/internal/live/r2.go new file mode 100644 index 0000000..b3d5928 --- /dev/null +++ b/cmd/acb-evolver/internal/live/r2.go @@ -0,0 +1,125 @@ +// Package live provides R2 upload for the evolution live.json feed. +package live + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// R2Config holds R2 configuration for live.json uploads. +type R2Config struct { + AccessKey string + SecretKey string + Endpoint string + Bucket string +} + +// R2ConfigFromEnv loads R2 configuration from environment variables. +func R2ConfigFromEnv() *R2Config { + return &R2Config{ + AccessKey: os.Getenv("ACB_R2_ACCESS_KEY"), + SecretKey: os.Getenv("ACB_R2_SECRET_KEY"), + Endpoint: getEnvOrDefault("ACB_R2_ENDPOINT", ""), + Bucket: os.Getenv("ACB_R2_BUCKET"), + } +} + +// HasCredentials returns true if R2 credentials are configured. +func (c *R2Config) HasCredentials() bool { + return c.AccessKey != "" && c.SecretKey != "" && c.Endpoint != "" && c.Bucket != "" +} + +// R2Client handles R2 bucket operations for live.json uploads. +type R2Client struct { + client *s3.Client + bucket string + endpoint string +} + +// NewR2Client creates a new R2 client. +func NewR2Client(cfg *R2Config) (*R2Client, error) { + if !cfg.HasCredentials() { + return nil, fmt.Errorf("R2 credentials not configured") + } + + // Create custom endpoint resolver for R2 + customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: cfg.Endpoint, + SigningRegion: "auto", + }, nil + }) + + // Load AWS config with R2 credentials + awsCfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + cfg.AccessKey, + cfg.SecretKey, + "", + )), + config.WithEndpointResolverWithOptions(customResolver), + ) + if err != nil { + return nil, fmt.Errorf("failed to load AWS config: %w", err) + } + + return &R2Client{ + client: s3.NewFromConfig(awsCfg), + bucket: cfg.Bucket, + endpoint: cfg.Endpoint, + }, nil +} + +// UploadLiveJSON uploads the live.json data to R2 at evolution/live.json. +// The file is served with Cache-Control: max-age=10 for near-real-time updates. +func (c *R2Client) UploadLiveJSON(ctx context.Context, data *LiveData) error { + b, err := json.MarshalIndent(data, "", " ") + if err != nil { + return fmt.Errorf("marshal: %w", err) + } + + _, err = c.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String("evolution/live.json"), + Body: bytes.NewReader(b), + ContentType: aws.String("application/json"), + CacheControl: aws.String("public, max-age=10"), + }) + if err != nil { + return fmt.Errorf("put object: %w", err) + } + + return nil +} + +// Upload uploads data to R2 at the specified key. +func (c *R2Client) Upload(ctx context.Context, key string, data []byte, contentType string) error { + _, err := c.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(key), + Body: bytes.NewReader(data), + ContentType: aws.String(contentType), + CacheControl: aws.String("public, max-age=10"), + }) + return err +} + +// Endpoint returns the R2 endpoint URL. +func (c *R2Client) Endpoint() string { + return c.endpoint +} + +func getEnvOrDefault(key, defaultValue string) string { + if val := os.Getenv(key); val != "" { + return val + } + return defaultValue +} diff --git a/cmd/acb-evolver/internal/live/r2_test.go b/cmd/acb-evolver/internal/live/r2_test.go new file mode 100644 index 0000000..c2823d1 --- /dev/null +++ b/cmd/acb-evolver/internal/live/r2_test.go @@ -0,0 +1,116 @@ +// Package live provides R2 upload for the evolution live.json feed. +package live + +import ( + "strings" + "testing" +) + +func TestR2ConfigFromEnv(t *testing.T) { + // Test with no env vars + cfg := R2ConfigFromEnv() + if cfg.AccessKey != "" { + t.Error("expected empty AccessKey") + } + if cfg.HasCredentials() { + t.Error("expected no credentials without env vars") + } +} + +func TestR2ConfigHasCredentials(t *testing.T) { + tests := []struct { + name string + accessKey string + secretKey string + endpoint string + bucket string + want bool + }{ + { + name: "all set", + accessKey: "key", + secretKey: "secret", + endpoint: "https://example.com", + bucket: "bucket", + want: true, + }, + { + name: "missing access key", + accessKey: "", + secretKey: "secret", + endpoint: "https://example.com", + bucket: "bucket", + want: false, + }, + { + name: "missing secret key", + accessKey: "key", + secretKey: "", + endpoint: "https://example.com", + bucket: "bucket", + want: false, + }, + { + name: "missing endpoint", + accessKey: "key", + secretKey: "secret", + endpoint: "", + bucket: "bucket", + want: false, + }, + { + name: "missing bucket", + accessKey: "key", + secretKey: "secret", + endpoint: "https://example.com", + bucket: "", + want: false, + }, + { + name: "all empty", + accessKey: "", + secretKey: "", + endpoint: "", + bucket: "", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &R2Config{ + AccessKey: tt.accessKey, + SecretKey: tt.secretKey, + Endpoint: tt.endpoint, + Bucket: tt.bucket, + } + if got := cfg.HasCredentials(); got != tt.want { + t.Errorf("HasCredentials() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNewR2Client_MissingCredentials(t *testing.T) { + cfg := &R2Config{} // no credentials + _, err := NewR2Client(cfg) + if err == nil { + t.Error("expected error for missing credentials") + } + if !strings.Contains(err.Error(), "credentials not configured") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestGetEnvOrDefault(t *testing.T) { + // Test with env var set + t.Setenv("TEST_VAR", "value") + if got := getEnvOrDefault("TEST_VAR", "default"); got != "value" { + t.Errorf("got %q, want %q", got, "value") + } + + // Test with env var not set + if got := getEnvOrDefault("NONEXISTENT_VAR", "default"); got != "default" { + t.Errorf("got %q, want %q", got, "default") + } +} diff --git a/cmd/acb-evolver/main.go b/cmd/acb-evolver/main.go index b131f3c..b953657 100644 --- a/cmd/acb-evolver/main.go +++ b/cmd/acb-evolver/main.go @@ -484,10 +484,12 @@ func runValidationStats(ctx context.Context, store *evolverdb.Store) { // runLiveExport exports the current evolution state to live.json. // -// live-export [-out evolution/live.json] +// live-export [-out evolution/live.json] [-r2] [-r2-only] func runLiveExport(ctx context.Context, db *sql.DB, args []string) { fs := flag.NewFlagSet("live-export", flag.ExitOnError) out := fs.String("out", envOrDefault("ACB_EVOLUTION_OUT", "evolution/live.json"), "output file path") + uploadR2 := fs.Bool("r2", false, "upload to R2 in addition to writing local file") + r2Only := fs.Bool("r2-only", false, "upload to R2 only, skip local file") if err := fs.Parse(args); err != nil { os.Exit(1) } @@ -496,11 +498,31 @@ func runLiveExport(ctx context.Context, db *sql.DB, args []string) { if err != nil { log.Fatalf("live-export: %v", err) } - if err := live.WriteFile(data, *out); err != nil { - log.Fatalf("live-export write: %v", err) + + // Write local file unless r2-only is set + if !*r2Only { + if err := live.WriteFile(data, *out); err != nil { + log.Fatalf("live-export write: %v", err) + } + log.Printf("live-export: wrote %d programs (%d promoted) to %s", + data.TotalPrograms, data.PromotedCount, *out) + } + + // Upload to R2 if requested + if *uploadR2 || *r2Only { + r2Cfg := live.R2ConfigFromEnv() + if !r2Cfg.HasCredentials() { + log.Fatalf("live-export: R2 credentials not configured (set ACB_R2_ACCESS_KEY, ACB_R2_SECRET_KEY, ACB_R2_ENDPOINT, ACB_R2_BUCKET)") + } + r2Client, err := live.NewR2Client(r2Cfg) + if err != nil { + log.Fatalf("live-export: create R2 client: %v", err) + } + if err := r2Client.UploadLiveJSON(ctx, data); err != nil { + log.Fatalf("live-export: upload to R2: %v", err) + } + log.Printf("live-export: uploaded to R2 at evolution/live.json (%d programs)", data.TotalPrograms) } - log.Printf("live-export: wrote %d programs (%d promoted) to %s", - data.TotalPrograms, data.PromotedCount, *out) } func mustOpenDB(url string) *sql.DB { diff --git a/web/src/api-types.ts b/web/src/api-types.ts index ca9b231..93f27b2 100644 --- a/web/src/api-types.ts +++ b/web/src/api-types.ts @@ -208,8 +208,13 @@ export async function registerBot(request: RegisterRequest): Promise { - const response = await fetch('/data/evolution/live.json'); + // Fetch from R2 for real-time updates (not from static Pages) + const response = await fetch(`${R2_BASE_URL}/evolution/live.json`); if (!response.ok) throw new Error(`Failed to fetch evolution data: ${response.status}`); return response.json(); }