diff --git a/benches/baselines/README.md b/benches/baselines/README.md new file mode 100644 index 0000000..9fbbba7 --- /dev/null +++ b/benches/baselines/README.md @@ -0,0 +1,145 @@ +# Baseline Metrics + +This directory stores baseline benchmark metrics for pdftract performance validation and regression tracking. + +## Purpose + +Baseline metrics serve as the reference point for: +- **Performance regression detection** - Compare current benchmark results against historical baselines +- **Competitive analysis** - Track pdftract performance relative to pdfminer.six, pypdf, and pdfplumber +- **CI/CD gates** - Block releases that introduce performance regressions beyond acceptable thresholds +- **Trend analysis** - Monitor performance improvements over time + +## File Format + +Each baseline file is a JSON document conforming to `schema.json`. The schema defines the following structure: + +### Required Fields + +| Field | Type | Unit | Description | +|-------|------|------|-------------| +| `commit_sha` | string | - | Git commit SHA (or "main" for tracking branch) | +| `timestamp` | string | ISO 8601 | When the baseline was recorded | +| `pdftract_geomean` | number | seconds | Geometric mean extraction time across all fixtures (pdftract) | +| `grep_1000_mean_ms` | number | milliseconds | Mean time for 1000-PDF corpus search | + +### Optional Fields + +| Field | Type | Unit | Description | +|-------|------|------|-------------| +| `pdfminer_geomean` | number | seconds | Geometric mean extraction time (pdfminer.six) | +| `pypdf_geomean` | number | seconds | Geometric mean extraction time (pypdf) | +| `pdfplumber_geomean` | number | seconds | Geometric mean extraction time (pdfplumber) | +| `throughput_mb_per_sec` | number | MB/s | Aggregate throughput for grep-corpus benchmark | +| `files_per_sec` | number | files/second | Processing rate for grep-corpus benchmark | +| `total_runtime_sec` | number | seconds | Wall-clock time for complete benchmark suite | +| `corpus_size` | integer | count | Number of PDF files in test corpus | +| `notes` | string | - | Free-form contextual information | + +## Naming Convention + +Baseline files are named by their Git branch or tag: + +- `main.json` - Baseline for the main development branch +- `v0.1.0.json` - Baseline for release tag v0.1.0 +- `v0.2.0.json` - Baseline for release tag v0.2.0 + +## Schema Validation + +All baseline files should validate against `schema.json`: + +```bash +# Validate a baseline file (requires ajv-cli or similar) +npx ajv validate --strict=false -s schema.json -d main.json + +# Or use Python jsonschema +python - <<'PYTHON' +import jsonschema, json +with open('schema.json') as s, with open('main.json') as d: + jsonschema.validate(json.load(d), json.load(s)) +PYTHON +``` + +## Usage in CI + +The baseline metrics are used in CI to detect performance regressions: + +1. **Baseline comparison**: Each benchmark run compares results against the appropriate baseline file +2. **Threshold checks**: Regressions exceeding 10% for primary metrics block the PR +3. **Competitive ratios**: pdftract must maintain ≥ 10× speedup vs pdfminer.six and ≥ 5× vs pypdf + +## Performance Targets + +Based on the Primary Objectives in the project plan: + +| Metric | Target | Measurement | +|--------|--------|-------------| +| 100-page vector PDF, 4-core | < 3 seconds | `cargo bench`, `tests/fixtures/perf/` | +| 10-page scanned PDF (OCR) | < 30 seconds | includes Tesseract | +| Single-page extraction latency | < 150 ms p99 | wrk benchmark | +| Throughput vs pdfminer.six | ≥ 10× faster | Identical hardware | +| Throughput vs pypdf | ≥ 5× faster | Same benchmark suite | +| `pdftract grep` throughput | ≥ 50 MB/s | 1000-PDF corpus, 4-core | + +## Updating Baselines + +When to update a baseline: + +1. **After a major release** - Create a new baseline file tagged with the release version +2. **After accepted performance improvements** - Update `main.json` when improvements merge +3. **Never for regressions** - Regressions should block release, not update baselines + +Update process: + +```bash +# Run benchmarks to generate new baseline +cargo bench --bench grep_corpus | tee /tmp/bench-results.txt + +# Extract metrics and create/update baseline file +# (This step requires a helper script to parse benchmark output) + +# Validate against schema +npx ajv validate --strict=false -s schema.json -d main.json + +# Commit the updated baseline +git add benches/baselines/main.json +git commit -m "bench(bf-XXX): update main baseline after performance improvements" +``` + +## Example Baseline + +```json +{ + "commit_sha": "abc1234", + "timestamp": "2024-07-06T10:30:45Z", + "pdftract_geomean": 2.5, + "pdfminer_geomean": 28.0, + "pypdf_geomean": 15.0, + "pdfplumber_geomean": 32.0, + "grep_1000_mean_ms": 18.5, + "throughput_mb_per_sec": 87.3, + "files_per_sec": 920.0, + "total_runtime_sec": 1.09, + "corpus_size": 1000, + "notes": "Baseline for v0.2.0 release - OCR improvements and grep optimization" +} +``` + +## Regression Detection + +The benchmark harness compares current results against baselines and flags regressions: + +- **PASS**: All metrics within ±5% of baseline (acceptable variance) +- **WARN**: Metrics degraded 5–10% (logged, non-blocking) +- **FAIL**: Metrics degraded >10% (blocks PR merge) +- **IMPROVEMENT**: Metrics improved >5% (logged, consider baseline update) + +## Historical Context + +Baseline files form a historical record of pdftract's performance evolution. The `main.json` baseline represents the current state of the main branch, while tagged baselines (e.g., `v0.1.0.json`) capture performance at specific release points. + +This history enables: +- Long-term performance trend analysis +- Release-to-release comparison +- Identification of performance bottlenecks +- Validation of optimization efforts diff --git a/benches/baselines/schema.json b/benches/baselines/schema.json new file mode 100644 index 0000000..1d7955b --- /dev/null +++ b/benches/baselines/schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "PDFtract Baseline Metrics Schema", + "description": "Schema for baseline benchmark metrics stored in benches/baselines/", + "type": "object", + "required": ["commit_sha", "timestamp", "pdftract_geomean", "grep_1000_mean_ms"], + "properties": { + "commit_sha": { + "type": "string", + "description": "Git commit SHA (short or full) for the pdftract version used", + "pattern": "^[a-fA-F0-9]{7,40}$|main$", + "examples": ["abc1234", "deadbeef1234567890", "main"] + }, + "timestamp": { + "type": "string", + "format": "date-time", + "description": "ISO 8601 timestamp when the baseline was recorded", + "examples": ["2024-01-01T00:00:00Z", "2024-07-06T10:30:45+00:00"] + }, + "pdftract_geomean": { + "type": "number", + "minimum": 0, + "description": "Geometric mean extraction time in seconds across all fixtures for pdftract", + "unit": "seconds" + }, + "pdfminer_geomean": { + "type": "number", + "minimum": 0, + "description": "Geometric mean extraction time in seconds across all fixtures for pdfminer.six", + "unit": "seconds" + }, + "pypdf_geomean": { + "type": "number", + "minimum": 0, + "description": "Geometric mean extraction time in seconds across all fixtures for pypdf", + "unit": "seconds" + }, + "pdfplumber_geomean": { + "type": "number", + "minimum": 0, + "description": "Geometric mean extraction time in seconds across all fixtures for pdfplumber", + "unit": "seconds" + }, + "grep_1000_mean_ms": { + "type": "number", + "minimum": 0, + "description": "Mean time in milliseconds for searching across 1000-PDF corpus", + "unit": "milliseconds" + }, + "throughput_mb_per_sec": { + "type": "number", + "minimum": 0, + "description": "Aggregate throughput in megabytes per second for grep-corpus benchmark", + "unit": "MB/s" + }, + "files_per_sec": { + "type": "number", + "minimum": 0, + "description": "Files processed per second in grep-corpus benchmark", + "unit": "files/second" + }, + "total_runtime_sec": { + "type": "number", + "minimum": 0, + "description": "Total wall-clock runtime in seconds for the complete benchmark suite", + "unit": "seconds" + }, + "corpus_size": { + "type": "integer", + "minimum": 0, + "description": "Number of PDF files in the test corpus", + "unit": "count" + }, + "notes": { + "type": "string", + "description": "Free-form notes about this baseline (e.g., placeholder, experimental, environmental factors)" + } + }, + "additionalProperties": false, + "examples": [ + { + "commit_sha": "main", + "timestamp": "2024-01-01T00:00:00Z", + "pdftract_geomean": 10.0, + "pdfminer_geomean": 100.0, + "pypdf_geomean": 120.0, + "pdfplumber_geomean": 150.0, + "grep_1000_mean_ms": 50.0, + "throughput_mb_per_sec": 78.5, + "files_per_sec": 850.0, + "total_runtime_sec": 1.18, + "corpus_size": 1000, + "notes": "Baseline for v0.1.0 release" + } + ] +}