9456d8e231
Implements the conformance test runner pattern for all 10 SDKs as specified in the plan (line 3547). Each SDK now has a dedicated conformance test runner. Created: - tests/sdk-conformance/report-schema.json: JSON schema for conformance reports - docs/notes/sdk-conformance-runner.md: Pattern documentation and reference - crates/pdftract-cli/tests/conformance.rs: Rust cargo test target - tests/conformance/test_conformance.py: Python pytest harness - tests/conformance/conformance.test.ts: Node.js vitest runner - tests/conformance/conformance_test.go: Go go test runner - tests/conformance/ConformanceTest.java: Java JUnit 5 runner - tests/conformance/ConformanceTests.cs: .NET xUnit runner - tests/conformance/conformance.c: C standalone binary - tests/conformance/conformance_test.rb: Ruby minitest runner - tests/conformance/ConformanceTest.php: PHP PHPUnit runner - tests/conformance/ConformanceTests.swift: Swift XCTest runner All runners implement: - Loading of tests/sdk-conformance/cases.json - Execution of test cases with language-native method invocations - Comparison of results against expected values with numeric tolerances - Emission of machine-readable conformance-report.json - Non-zero exit on failures/errors for CI gating Acceptance criteria: - PASS: All 10 SDKs have language-specific runners - PASS: Runners consume shared cases.json - PASS: Runners emit JSON reports matching schema - PASS: Runners exit non-zero on failure - WARN: README integration pending SDK repo creation - WARN: Stub implementations return placeholder results References: - Plan line 3547: "Every SDK has a pdftract-sdk-conformance test runner" - Plan line 3589: "Conformance suite results published as Argo artifact" Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> Bead-Id: pdftract-5omc
123 lines
3.8 KiB
JSON
123 lines
3.8 KiB
JSON
{
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"$id": "https://github.com/jedarden/pdftract/schemas/conformance-report-v1.json",
|
|
"title": "pdftract SDK Conformance Report Schema",
|
|
"description": "Schema for conformance test reports produced by SDK conformance runners.",
|
|
"type": "object",
|
|
"required": ["sdk", "sdk_version", "suite_version", "timestamp", "results", "summary"],
|
|
"properties": {
|
|
"sdk": {
|
|
"type": "string",
|
|
"description": "SDK name (e.g., 'pdftract-py', 'pdftract-node', 'pdftract-rust')."
|
|
},
|
|
"sdk_version": {
|
|
"type": "string",
|
|
"description": "SDK version that produced this report.",
|
|
"pattern": "^\\d+\\.\\d+\\.\\d+(-[a-z0-9.]+)?$"
|
|
},
|
|
"suite_version": {
|
|
"type": "string",
|
|
"description": "Version of the conformance suite that was run.",
|
|
"pattern": "^\\d+\\.\\d+\\.\\d+$"
|
|
},
|
|
"schema_version": {
|
|
"type": "string",
|
|
"description": "Version of the pdftract output schema targeted.",
|
|
"pattern": "^\\d+\\.\\d+$"
|
|
},
|
|
"timestamp": {
|
|
"type": "string",
|
|
"description": "ISO 8601 timestamp when the report was generated.",
|
|
"format": "date-time"
|
|
},
|
|
"results": {
|
|
"type": "array",
|
|
"description": "Per-case test results.",
|
|
"items": {
|
|
"type": "object",
|
|
"required": ["id", "status"],
|
|
"properties": {
|
|
"id": {
|
|
"type": "string",
|
|
"description": "Test case ID from the suite."
|
|
},
|
|
"status": {
|
|
"type": "string",
|
|
"enum": ["pass", "fail", "skip", "error"],
|
|
"description": "Test result status."
|
|
},
|
|
"actual": {
|
|
"description": "Actual value returned by the SDK (for debugging)."
|
|
},
|
|
"expected": {
|
|
"description": "Expected value from the test case."
|
|
},
|
|
"error": {
|
|
"type": "string",
|
|
"description": "Error message (for status='error')."
|
|
},
|
|
"reason": {
|
|
"type": "string",
|
|
"description": "Human-readable reason for failure or skip."
|
|
},
|
|
"duration_ms": {
|
|
"type": "number",
|
|
"description": "Test execution time in milliseconds."
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"summary": {
|
|
"type": "object",
|
|
"required": ["total", "passed", "failed", "skipped", "errors"],
|
|
"properties": {
|
|
"total": {
|
|
"type": "integer",
|
|
"description": "Total number of test cases."
|
|
},
|
|
"passed": {
|
|
"type": "integer",
|
|
"description": "Number of passed tests."
|
|
},
|
|
"failed": {
|
|
"type": "integer",
|
|
"description": "Number of failed tests."
|
|
},
|
|
"skipped": {
|
|
"type": "integer",
|
|
"description": "Number of skipped tests (feature unavailable, schema version mismatch)."
|
|
},
|
|
"errors": {
|
|
"type": "integer",
|
|
"description": "Number of tests that errored (exception, crash)."
|
|
},
|
|
"duration_ms": {
|
|
"type": "number",
|
|
"description": "Total execution time in milliseconds."
|
|
}
|
|
}
|
|
},
|
|
"environment": {
|
|
"type": "object",
|
|
"description": "Optional environment information for debugging.",
|
|
"properties": {
|
|
"os": {
|
|
"type": "string",
|
|
"description": "Operating system."
|
|
},
|
|
"arch": {
|
|
"type": "string",
|
|
"description": "Architecture (e.g., 'x86_64', 'aarch64')."
|
|
},
|
|
"binary_version": {
|
|
"type": "string",
|
|
"description": "Version of the pdftract binary invoked."
|
|
},
|
|
"runtime_version": {
|
|
"type": "string",
|
|
"description": "Language runtime version (e.g., 'Python 3.12.0', 'Node.js v20.10.0')."
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|