groups: - name: pdftract_serve interval: 30s rules: - alert: PdftractExtractionLatencyHigh expr: | histogram_quantile(0.99, sum(rate(pdftract_extraction_duration_seconds_bucket[5m])) by (le, instance) ) > 0.3 for: 5m labels: severity: warning service: pdftract annotations: summary: "Extraction latency p99 above 300ms threshold (2× target)" description: > The 99th percentile extraction duration is {{ $value }}s, exceeding 300ms (2× the 150ms target) for 5 minutes. This indicates degraded performance. runbook: "https://github.com/jedarden/pdftract/blob/main/docs/plan/plan.md#L3832" - alert: PdftractErrorRateHigh expr: | sum(rate(pdftract_http_requests_total{status=~"5.."}[5m])) / sum(rate(pdftract_http_requests_total[5m])) > 0.01 for: 5m labels: severity: critical service: pdftract annotations: summary: "HTTP error rate above 1% threshold" description: > Error rate is {{ $value | humanizePercentage }}, exceeding 1% threshold for 5 minutes. This indicates a significant service degradation or failure. runbook: "https://github.com/jedarden/pdftract/blob/main/docs/plan/plan.md#L3832" - alert: PdftractCacheHitRateLow expr: | ( sum(rate(pdftract_cache_hits_total[5m])) / (sum(rate(pdftract_cache_hits_total[5m])) + sum(rate(pdftract_cache_misses_total[5m]))) ) < 0.5 for: 10m labels: severity: info service: pdftract annotations: summary: "Cache hit rate below 50% threshold" description: > Cache hit rate is {{ $value | humanizePercentage }}, below 50% threshold for 10 minutes. This may indicate cache configuration issues or insufficient cache capacity. runbook: "https://github.com/jedarden/pdftract/blob/main/docs/plan/plan.md#L3832" - alert: PdftractWorkerSaturated expr: | pdftract_rayon_pool_utilization > 0.8 for: 5m labels: severity: warning service: pdftract annotations: summary: "Worker pool utilization above 80% threshold" description: > Rayon worker pool utilization is {{ $value | humanizePercentage }}, above 80% threshold for 5 minutes. The service is at risk of becoming unresponsive under load. runbook: "https://github.com/jedarden/pdftract/blob/main/docs/plan/plan.md#L3832" - alert: PdftractMemoryHigh expr: | ( process_resident_memory_bytes{job="pdftract-serve"} / (pdftract_max_decompress_bytes * 0.8) ) > 1 for: 5m labels: severity: warning service: pdftract annotations: summary: "Memory usage above 80% of max_decompress_bytes ceiling" description: > RSS memory is {{ $value | humanize }}, exceeding 80% of the configured max_decompress_bytes ceiling. This may lead to OOM under load. runbook: "https://github.com/jedarden/pdftract/blob/main/docs/plan/plan.md#L3832" # Additional alert from plan recommendations - alert: PdftractCacheSizeGrowingUnchecked expr: | abs(deriv(pdftract_cache_size_bytes[1h])) > 1e9 for: 6h labels: severity: warning service: pdftract annotations: summary: "Cache size growing faster than 1 GB/hour" description: > Cache size is changing by more than 1 GB/hour (current rate: {{ $value | humanize }}B/h) for 6 hours. This may indicate a cache eviction or configuration problem. runbook: "https://github.com/jedarden/pdftract/blob/main/docs/plan/plan.md#L3832" - alert: PdftractDiagnosticFlood expr: | sum(rate(pdftract_diagnostic_emitted_total{severity="error"}[5m])) > 10 for: 5m labels: severity: warning service: pdftract annotations: summary: "Error diagnostic rate above 10/sec" description: > Error diagnostics are being emitted at {{ $value }}/sec, indicating a potential software bug or infrastructure issue. runbook: "https://github.com/jedarden/pdftract/blob/main/docs/plan/plan.md#L3832"