From f716d27aac07c81fd2c2f56f3e65feaa3d0a81c6 Mon Sep 17 00:00:00 2001
From: jedarden <github@jedarden.com>
Date: Sun, 5 Jul 2026 12:49:49 -0400
Subject: [PATCH] docs(bf-57ko1): add Prometheus alerting rules for serve
 deployments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Add 7 alerting rules for pdftract serve monitoring
- Required alerts per plan Monitoring section:
  - PdftractExtractionLatencyHigh: p99 > 2× 150ms target for >5m
  - PdftractErrorRateHigh: error rate > 1% over 5m
  - PdftractCacheHitRateLow: cache hit rate < 50%
  - PdftractWorkerSaturated: worker pool > 80% busy
  - PdftractMemoryHigh: RSS > 80% of max_decompress_bytes
- Additional alerts from plan recommendations:
  - PdftractCacheSizeGrowingUnchecked
  - PdftractDiagnosticFlood
- Runbook URLs reference plan.md monitoring section

Acceptance criteria:
- ✓ Valid Prometheus YAML structure
- ✓ Contains 7 alerting rules (5 required + 2 additional)
- ✓ Runbook URLs point to docs/plan/plan.md monitoring section

Closes bf-57ko1
---
 docs/operations/prometheus-rules.yaml | 114 ++++++++++++++++++++++++++
 1 file changed, 114 insertions(+)
 create mode 100644 docs/operations/prometheus-rules.yaml

diff --git a/docs/operations/prometheus-rules.yaml b/docs/operations/prometheus-rules.yaml
new file mode 100644
index 0000000..c518d07
--- /dev/null
+++ b/docs/operations/prometheus-rules.yaml
@@ -0,0 +1,114 @@
+groups:
+  - name: pdftract_serve
+    interval: 30s
+    rules:
+      - alert: PdftractExtractionLatencyHigh
+        expr: |
+          histogram_quantile(0.99,
+            sum(rate(pdftract_extraction_duration_seconds_bucket[5m])) by (le, instance)
+          ) > 0.3
+        for: 5m
+        labels:
+          severity: warning
+          service: pdftract
+        annotations:
+          summary: "Extraction latency p99 above 300ms threshold (2× target)"
+          description: >
+            The 99th percentile extraction duration is {{ $value }}s, exceeding 300ms
+            (2× the 150ms target) for 5 minutes. This indicates degraded performance.
+          runbook: "https://github.com/jedarden/pdftract/blob/main/docs/plan/plan.md#L3832"
+
+      - alert: PdftractErrorRateHigh
+        expr: |
+          sum(rate(pdftract_http_requests_total{status=~"5.."}[5m])) /
+          sum(rate(pdftract_http_requests_total[5m])) > 0.01
+        for: 5m
+        labels:
+          severity: critical
+          service: pdftract
+        annotations:
+          summary: "HTTP error rate above 1% threshold"
+          description: >
+            Error rate is {{ $value | humanizePercentage }}, exceeding 1% threshold
+            for 5 minutes. This indicates a significant service degradation or failure.
+          runbook: "https://github.com/jedarden/pdftract/blob/main/docs/plan/plan.md#L3832"
+
+      - alert: PdftractCacheHitRateLow
+        expr: |
+          (
+            sum(rate(pdftract_cache_hits_total[5m])) /
+            (sum(rate(pdftract_cache_hits_total[5m])) + sum(rate(pdftract_cache_misses_total[5m])))
+          ) < 0.5
+        for: 10m
+        labels:
+          severity: info
+          service: pdftract
+        annotations:
+          summary: "Cache hit rate below 50% threshold"
+          description: >
+            Cache hit rate is {{ $value | humanizePercentage }}, below 50% threshold
+            for 10 minutes. This may indicate cache configuration issues or
+            insufficient cache capacity.
+          runbook: "https://github.com/jedarden/pdftract/blob/main/docs/plan/plan.md#L3832"
+
+      - alert: PdftractWorkerSaturated
+        expr: |
+          pdftract_rayon_pool_utilization > 0.8
+        for: 5m
+        labels:
+          severity: warning
+          service: pdftract
+        annotations:
+          summary: "Worker pool utilization above 80% threshold"
+          description: >
+            Rayon worker pool utilization is {{ $value | humanizePercentage }},
+            above 80% threshold for 5 minutes. The service is at risk of becoming
+            unresponsive under load.
+          runbook: "https://github.com/jedarden/pdftract/blob/main/docs/plan/plan.md#L3832"
+
+      - alert: PdftractMemoryHigh
+        expr: |
+          (
+            process_resident_memory_bytes{job="pdftract-serve"} /
+            (pdftract_max_decompress_bytes * 0.8)
+          ) > 1
+        for: 5m
+        labels:
+          severity: warning
+          service: pdftract
+        annotations:
+          summary: "Memory usage above 80% of max_decompress_bytes ceiling"
+          description: >
+            RSS memory is {{ $value | humanize }}, exceeding 80% of the configured
+            max_decompress_bytes ceiling. This may lead to OOM under load.
+          runbook: "https://github.com/jedarden/pdftract/blob/main/docs/plan/plan.md#L3832"
+
+      # Additional alert from plan recommendations
+      - alert: PdftractCacheSizeGrowingUnchecked
+        expr: |
+          abs(deriv(pdftract_cache_size_bytes[1h])) > 1e9
+        for: 6h
+        labels:
+          severity: warning
+          service: pdftract
+        annotations:
+          summary: "Cache size growing faster than 1 GB/hour"
+          description: >
+            Cache size is changing by more than 1 GB/hour (current rate:
+            {{ $value | humanize }}B/h) for 6 hours. This may indicate a
+            cache eviction or configuration problem.
+          runbook: "https://github.com/jedarden/pdftract/blob/main/docs/plan/plan.md#L3832"
+
+      - alert: PdftractDiagnosticFlood
+        expr: |
+          sum(rate(pdftract_diagnostic_emitted_total{severity="error"}[5m])) > 10
+        for: 5m
+        labels:
+          severity: warning
+          service: pdftract
+        annotations:
+          summary: "Error diagnostic rate above 10/sec"
+          description: >
+            Error diagnostics are being emitted at {{ $value }}/sec, indicating
+            a potential software bug or infrastructure issue.
+          runbook: "https://github.com/jedarden/pdftract/blob/main/docs/plan/plan.md#L3832"