- Add Exploration and Formation axis definitions with feature extraction from source code pattern matching (exploration/formation indicators) - Extend Grid key from (x,y) to (x,y,z,w) with 3⁴=81-cell behavior grid - Update bin assignment, promotion gate, and persistence (JSON snapshot) - Add Slice() for 2-D dashboard visualization across any axis pair - Migration: old 2-D archives project at z=middle, w=middle - Update cross-pollination to pad 2-element behavior vectors to 4 - Add Prometheus metrics to matchmaker (bot crashes, stale job count) - Add rivalry detection to index builder (data/meta/rivalries.json) - Web: batched bot list loading, leaderboard keyboard accessibility, improved ARIA attributes on match/playlist cards Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
135 lines
4.3 KiB
Go
135 lines
4.3 KiB
Go
// Package metrics defines Prometheus metrics for AI Code Battle services per plan §9.9.
|
|
//
|
|
// All services import this package to expose a /metrics endpoint on an
|
|
// internal port (default :9090). The metrics match the 9 monitoring signals
|
|
// listed in the plan.
|
|
package metrics
|
|
|
|
import (
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
|
)
|
|
|
|
// §9.9 metric definitions — registered once at init time.
|
|
var (
|
|
// MatchThroughput counts completed matches (worker increments per result).
|
|
MatchThroughput = prometheus.NewCounter(prometheus.CounterOpts{
|
|
Name: "acb_match_throughput_total",
|
|
Help: "Total number of matches completed.",
|
|
})
|
|
|
|
// JobQueueDepth tracks the Valkey job queue length (matchmaker updates each tick).
|
|
JobQueueDepth = prometheus.NewGauge(prometheus.GaugeOpts{
|
|
Name: "acb_job_queue_depth",
|
|
Help: "Current number of pending jobs in the Valkey queue.",
|
|
})
|
|
|
|
// BotCrashed counts bots marked as crashed by the health checker.
|
|
BotCrashed = prometheus.NewCounter(prometheus.CounterOpts{
|
|
Name: "acb_bot_crashed_total",
|
|
Help: "Total number of bot crash events detected by the health checker.",
|
|
})
|
|
|
|
// StaleJobCount is the number of stale jobs found in the last reaper cycle.
|
|
StaleJobCount = prometheus.NewGauge(prometheus.GaugeOpts{
|
|
Name: "acb_job_stale_count",
|
|
Help: "Number of stale jobs found in the most recent reaper cycle.",
|
|
})
|
|
|
|
// R2BytesUsed tracks the R2 warm cache size in bytes (index-builder updates).
|
|
R2BytesUsed = prometheus.NewGauge(prometheus.GaugeOpts{
|
|
Name: "acb_r2_bytes_used",
|
|
Help: "Total bytes used in the R2 warm cache.",
|
|
})
|
|
|
|
// ReplayUploadLatency tracks B2 replay upload duration.
|
|
ReplayUploadLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
|
|
Name: "acb_replay_upload_latency_seconds",
|
|
Help: "Latency of replay uploads to B2 in seconds.",
|
|
Buckets: prometheus.DefBuckets,
|
|
})
|
|
|
|
// EvolverGenerations counts evolution cycles completed.
|
|
EvolverGenerations = prometheus.NewCounter(prometheus.CounterOpts{
|
|
Name: "acb_evolver_generations_total",
|
|
Help: "Total number of evolution generations completed.",
|
|
})
|
|
|
|
// IndexBuildDuration tracks how long each index build cycle takes.
|
|
IndexBuildDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
|
|
Name: "acb_index_build_duration_seconds",
|
|
Help: "Duration of index build cycles in seconds.",
|
|
Buckets: []float64{1, 5, 10, 30, 60, 120, 300, 600},
|
|
})
|
|
|
|
// HTTPRequestsTotal counts HTTP requests served by the API.
|
|
HTTPRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "acb_http_requests_total",
|
|
Help: "Total number of HTTP requests served.",
|
|
}, []string{"method", "path", "status"})
|
|
)
|
|
|
|
func init() {
|
|
prometheus.MustRegister(
|
|
MatchThroughput,
|
|
JobQueueDepth,
|
|
BotCrashed,
|
|
StaleJobCount,
|
|
R2BytesUsed,
|
|
ReplayUploadLatency,
|
|
EvolverGenerations,
|
|
IndexBuildDuration,
|
|
HTTPRequestsTotal,
|
|
)
|
|
}
|
|
|
|
// Handler returns an http.Handler that serves /metrics.
|
|
func Handler() http.Handler {
|
|
mux := http.NewServeMux()
|
|
mux.Handle("/metrics", promhttp.Handler())
|
|
mux.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
w.Write([]byte(`{"status":"ok"}`))
|
|
})
|
|
return mux
|
|
}
|
|
|
|
// StartServer starts a Prometheus metrics HTTP server. Returns the server
|
|
// so the caller can shut it down gracefully. The address defaults to
|
|
// ACB_METRICS_ADDR env var, falling back to ":9090".
|
|
func StartServer() *http.Server {
|
|
addr := os.Getenv("ACB_METRICS_ADDR")
|
|
if addr == "" {
|
|
addr = ":9090"
|
|
}
|
|
srv := &http.Server{Addr: addr, Handler: Handler()}
|
|
go srv.ListenAndServe()
|
|
return srv
|
|
}
|
|
|
|
// HTTPMiddleware wraps an http.Handler to count requests via HTTPRequestsTotal.
|
|
func HTTPMiddleware(next http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
start := time.Now()
|
|
sw := &statusWriter{ResponseWriter: w, status: http.StatusOK}
|
|
next.ServeHTTP(sw, r)
|
|
HTTPRequestsTotal.WithLabelValues(r.Method, r.URL.Path, strconv.Itoa(sw.status)).Inc()
|
|
_ = start
|
|
})
|
|
}
|
|
|
|
// statusWriter wraps http.ResponseWriter to capture the status code.
|
|
type statusWriter struct {
|
|
http.ResponseWriter
|
|
status int
|
|
}
|
|
|
|
func (w *statusWriter) WriteHeader(code int) {
|
|
w.status = code
|
|
w.ResponseWriter.WriteHeader(code)
|
|
}
|