pdftract/crates/pdftract-cli/src/main.rs
jedarden 857f928732 feat(pdftract-5omc): implement SDK conformance test runner pattern
Implement the conformance test runner pattern that every SDK will
implement to validate against the shared test suite.

- Rust reference implementation (crates/pdftract-core/tests/conformance.rs)
  * Full test suite loader and executor
  * Comparison engine with min/max, string constraints, tolerances
  * Skip logic for unsupported features and schema versions
  * Report generation in JSON format

- CLI compare subcommand (crates/pdftract-cli/src/main.rs)
  * pdftract compare - Compare actual vs expected with tolerances
  * Cross-language comparison tool to avoid reimplementations

- Documentation (docs/conformance/sdk-contract.md)
  * Complete pattern specification with pseudocode
  * Per-language runner locations
  * CI integration requirements

- Python reference stub (tests/python-conformance/test_conformance.py)
  * Full pytest-based implementation following the pattern

Closes: pdftract-5omc
2026-05-18 01:22:23 -04:00

391 lines
13 KiB
Rust

use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use std::fs;
use std::path::PathBuf;
#[derive(Parser)]
#[command(name = "pdftract")]
#[command(about = "pdftract CLI - PDF extraction and conformance testing", long_about = None)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
/// Compare actual results against expected values with tolerances (for conformance testing)
Compare {
/// Path to the actual results JSON
actual: PathBuf,
/// Path to the expected results JSON
expected: PathBuf,
/// Path to the tolerances JSON (optional)
#[arg(short, long)]
tolerances: Option<PathBuf>,
/// Output format (text, json)
#[arg(short, long, default_value = "text")]
format: String,
},
/// Run SDK conformance test suite
Conformance {
/// Path to the conformance suite JSON
#[arg(short, long, default_value = "tests/sdk-conformance/cases.json")]
suite: PathBuf,
/// SDK name
#[arg(short, long, default_value = "pdftract")]
sdk: String,
/// SDK version
#[arg(short, long, default_value = "0.1.0")]
version: String,
/// Output report path
#[arg(short, long, default_value = "conformance-report.json")]
output: PathBuf,
},
}
fn main() -> Result<()> {
let cli = Cli::parse();
match cli.command {
Commands::Compare {
actual,
expected,
tolerances,
format,
} => {
cmd_compare(actual, expected, tolerances, &format)?;
}
Commands::Conformance {
suite,
sdk,
version,
output,
} => {
cmd_conformance(suite, &sdk, &version, output)?;
}
}
Ok(())
}
fn cmd_compare(actual: PathBuf, expected: PathBuf, tolerances: Option<PathBuf>, format: &str) -> Result<()> {
let actual_json = fs::read_to_string(&actual)
.context(format!("Failed to read actual results from {:?}", actual))?;
let actual_val: serde_json::Value = serde_json::from_str(&actual_json)
.context("Failed to parse actual results as JSON")?;
let expected_json = fs::read_to_string(&expected)
.context(format!("Failed to read expected results from {:?}", expected))?;
let expected_val: serde_json::Value = serde_json::from_str(&expected_json)
.context("Failed to parse expected results as JSON")?;
let tolerances_val = if let Some(tol_path) = tolerances {
let tol_json = fs::read_to_string(&tol_path)
.context(format!("Failed to read tolerances from {:?}", tol_path))?;
Some(serde_json::from_str::<serde_json::Value>(&tol_json)
.context("Failed to parse tolerances as JSON")?)
} else {
None
};
let result = compare_values(&actual_val, &expected_val, tolerances_val.as_ref())?;
match format {
"json" => {
let output = serde_json::to_string_pretty(&result)?;
println!("{}", output);
}
_ => {
print_compare_result(&result);
}
}
Ok(())
}
fn cmd_conformance(suite: PathBuf, sdk: &str, version: &str, output: PathBuf) -> Result<()> {
println!("Running conformance suite: {:?}", suite);
println!("SDK: {} v{}", sdk, version);
println!("Output: {:?}", output);
let suite_json = fs::read_to_string(&suite)
.context(format!("Failed to read suite from {:?}", suite))?;
let suite_val: serde_json::Value = serde_json::from_str(&suite_json)
.context("Failed to parse suite as JSON")?;
let cases = suite_val
.get("cases")
.and_then(|v| v.as_array())
.context("Suite missing 'cases' array")?;
println!("\nFound {} test cases", cases.len());
// This is a stub - actual implementation would invoke the SDK
let results: Vec<serde_json::Value> = cases
.iter()
.map(|case| {
serde_json::json!({
"id": case.get("id").unwrap_or(&serde_json::json!("unknown")),
"status": "skip",
"error": "SDK conformance runner not yet implemented - use language-specific runner"
})
})
.collect();
let report = serde_json::json!({
"sdk": sdk,
"sdk_version": version,
"suite_version": suite_val.get("version").unwrap_or(&serde_json::json!("unknown")),
"timestamp": chrono::Utc::now().to_rfc3339(),
"results": results,
"summary": {
"total": results.len(),
"passed": 0,
"failed": 0,
"skipped": results.len(),
"errors": 0
}
});
fs::write(&output, serde_json::to_string_pretty(&report)?)
.context(format!("Failed to write report to {:?}", output))?;
println!("\nReport written to {:?}", output);
Ok(())
}
#[derive(Debug, serde::Serialize)]
enum CompareResult {
Pass,
Fail { reason: String },
Missing,
}
fn compare_values(
actual: &serde_json::Value,
expected: &serde_json::Value,
tolerances: Option<&serde_json::Value>,
) -> Result<std::collections::HashMap<String, CompareResult>> {
let mut results = std::collections::HashMap::new();
compare_recursive(actual, expected, tolerances, "", &mut results);
Ok(results)
}
fn compare_recursive(
actual: &serde_json::Value,
expected: &serde_json::Value,
tolerances: Option<&serde_json::Value>,
path: &str,
results: &mut std::collections::HashMap<String, CompareResult>,
) {
match (actual, expected) {
// Handle min/max constraints
(serde_json::Value::Number(act), serde_json::Value::Object(exp)) => {
if let Some(min) = exp.get("min").and_then(|v| v.as_i64()) {
if act.as_i64().map_or(true, |v| v < min) {
results.insert(
path.to_string(),
CompareResult::Fail {
reason: format!("value {} is less than minimum {}", act, min),
},
);
return;
}
}
if let Some(max) = exp.get("max").and_then(|v| v.as_i64()) {
if act.as_i64().map_or(true, |v| v > max) {
results.insert(
path.to_string(),
CompareResult::Fail {
reason: format!("value {} is greater than maximum {}", act, max),
},
);
return;
}
}
if let Some(val) = exp.get("value") {
let tol = find_tolerance(tolerances, path);
let result = compare_with_tolerance(act, val, tol);
results.insert(path.to_string(), result);
} else {
results.insert(path.to_string(), CompareResult::Pass);
}
}
// String constraints
(serde_json::Value::String(act), serde_json::Value::Object(exp)) => {
if let Some(min_len) = exp.get("min_length").and_then(|v| v.as_usize()) {
if act.len() < min_len {
results.insert(
path.to_string(),
CompareResult::Fail {
reason: format!(
"string length {} is less than minimum {}",
act.len(),
min_len
),
},
);
return;
}
}
if let Some(containers) = exp.get("contains").and_then(|v| v.as_array()) {
for substring in containers {
if let Some(s) = substring.as_str() {
if !act.contains(s) {
results.insert(
path.to_string(),
CompareResult::Fail {
reason: format!("string does not contain '{}'", s),
},
);
return;
}
}
}
}
results.insert(path.to_string(), CompareResult::Pass);
}
// Array length constraints
(serde_json::Value::Array(act), serde_json::Value::Object(exp)) => {
if let Some(min_len) = exp.get("min").and_then(|v| v.as_usize()) {
if act.len() < min_len {
results.insert(
path.to_string(),
CompareResult::Fail {
reason: format!(
"array length {} is less than minimum {}",
act.len(),
min_len
),
},
);
return;
}
}
if let Some(max_len) = exp.get("max").and_then(|v| v.as_usize()) {
if act.len() > max_len {
results.insert(
path.to_string(),
CompareResult::Fail {
reason: format!(
"array length {} is greater than maximum {}",
act.len(),
max_len
),
},
);
return;
}
}
results.insert(path.to_string(), CompareResult::Pass);
}
// Direct comparison
(a, e) => {
if a == e {
results.insert(path.to_string(), CompareResult::Pass);
} else {
results.insert(
path.to_string(),
CompareResult::Fail {
reason: format!("expected {:?}, got {:?}", e, a),
},
);
}
}
}
}
fn compare_with_tolerance(
actual: &serde_json::Number,
expected: &serde_json::Value,
tolerance: Option<&serde_json::Value>,
) -> CompareResult {
let act_val = actual.as_f64().unwrap();
let exp_val = match expected {
serde_json::Value::Number(n) => n.as_f64().unwrap(),
_ => return CompareResult::Fail { reason: "expected value is not a number".to_string() },
};
if let Some(tol) = tolerance {
if let Some(obj) = tol.as_object() {
if let Some(abs_tol) = obj.get("abs").and_then(|v| v.as_f64()) {
let diff = (act_val - exp_val).abs();
if diff <= abs_tol {
return CompareResult::Pass;
}
}
if let Some(rel_tol) = obj.get("rel").and_then(|v| v.as_f64()) {
let diff = (act_val - exp_val).abs();
let avg = (act_val + exp_val) / 2.0;
if avg > 0.0 && diff / avg <= rel_tol {
return CompareResult::Pass;
}
}
}
}
// Direct comparison
if (act_val - exp_val).abs() < f64::EPSILON {
CompareResult::Pass
} else {
CompareResult::Fail {
reason: format!("numeric mismatch: {} vs {}", act_val, exp_val),
}
}
}
fn find_tolerance<'a>(
tolerances: Option<&'a serde_json::Value>,
path: &str,
) -> Option<&'a serde_json::Value> {
let tol = tolerances?;
if let Some(obj) = tol.as_object() {
// Try exact path match
if let Some(val) = obj.get(path) {
return Some(val);
}
// Try wildcard patterns
for (key, val) in obj {
if key.contains('*') {
let pattern = key.replace('*', ".*");
if let Ok(re) = regex::Regex::new(&pattern) {
if re.is_match(path) {
return Some(val);
}
}
}
}
}
None
}
fn print_compare_result(results: &std::collections::HashMap<String, CompareResult>) {
let mut passed = 0;
let mut failed = 0;
for (path, result) in results {
match result {
CompareResult::Pass => {
passed += 1;
}
CompareResult::Fail { reason } => {
failed += 1;
eprintln!("FAIL [{}]: {}", path, reason);
}
CompareResult::Missing => {
failed += 1;
eprintln!("MISSING [{}]: value not found in actual", path);
}
}
}
println!("\nComparison complete:");
println!(" Passed: {}", passed);
println!(" Failed: {}", failed);
if failed > 0 {
std::process::exit(1);
}
}