*/ declare(strict_types=1); namespace Pdftract\Tests; use PHPUnit\Framework\TestCase; class ConformanceTest extends TestCase { private const SUITE_PATH = 'tests/sdk-conformance/cases.json'; private const SDK_NAME = 'pdftract-php'; private const SDK_VERSION = '0.1.0'; private const STATUS_PASS = 'pass'; private const STATUS_FAIL = 'fail'; private const STATUS_SKIP = 'skip'; private const STATUS_ERROR = 'error'; /** * @dataProvider provideConformanceCases */ public function testConformanceCase(array $case, string $schemaVersion, string $fixturesBase): void { $result = $this->runTestCase($case, $schemaVersion, $fixturesBase); $this->addToAssertionCount(1); if ($result['status'] === self::STATUS_FAIL) { $this->fail($result['reason'] ?? 'Test failed'); } if ($result['status'] === self::STATUS_ERROR) { $this->fail($result['error'] ?? 'Test errored'); } } public function testConformanceSuite(): void { $suitePath = self::SUITE_PATH; $outputPath = 'conformance-report.json'; $report = $this->runConformance($suitePath, $outputPath); $this->assertEquals(0, $report['summary']['failed'], 'Some tests failed'); $this->assertEquals(0, $report['summary']['errors'], 'Some tests errored'); } private function compareWithTolerance(float $actual, float $expected, ?array $tolerance): bool { if ($tolerance === null) { return abs($actual - $expected) < PHP_FLOAT_EPSILON; } if (isset($tolerance['abs'])) { if (abs($actual - $expected) <= $tolerance['abs']) { return true; } } if (isset($tolerance['rel'])) { $diff = abs($actual - $expected); $avg = ($actual + $expected) / 2.0; if ($avg > 0.0 && $diff / $avg <= $tolerance['rel']) { return true; } } return false; } private function findTolerance(?array $tolerances, string $path): ?array { if ($tolerances === null) { return null; } if (isset($tolerances[$path])) { return $tolerances[$path]; } foreach ($tolerances as $key => $val) { if (str_contains($key, '*')) { $pattern = str_replace('*', '.*', $key); if (preg_match('/^' . $pattern . '$/', $path)) { return $val; } } } return null; } /** * @return array{passed: bool, reason: string|null} */ private function compareResults($actual, $expected, ?array $tolerances, string $path = ''): array { if (is_array($expected) && isset($expected['min'])) { if (is_numeric($actual)) { if ($actual < $expected['min']) { return [false, "{$path}: value {$actual} < minimum {$expected['min']}"]; } } } if (is_array($expected) && isset($expected['max'])) { if (is_numeric($actual)) { if ($actual > $expected['max']) { return [false, "{$path}: value {$actual} > maximum {$expected['max']}"]; } } } if (is_array($expected) && isset($expected['value'])) { if (is_numeric($actual)) { $tol = $this->findTolerance($tolerances, $path); if (!$this->compareWithTolerance((float)$actual, (float)$expected['value'], $tol)) { return [false, "{$path}: numeric mismatch"]; } } } if (is_array($expected) && isset($expected['min_length'])) { if (is_string($actual)) { if (strlen($actual) < $expected['min_length']) { return [false, "{$path}: string length too short"]; } } } if (is_array($expected) && isset($expected['contains'])) { if (is_string($actual)) { foreach ($expected['contains'] as $substring) { if (!str_contains($actual, $substring)) { return [false, "{$path}: string does not contain '{$substring}'"]; } } } } if (is_array($expected) && is_array($actual)) { foreach ($expected as $key => $expVal) { if ($key === 'min' || $key === 'max' || $key === 'value' || $key === 'min_length' || $key === 'contains') { continue; } $newPath = $path === '' ? $key : "{$path}.{$key}"; if (!array_key_exists($key, $actual)) { return [false, "{$newPath}: missing key '{$key}'"]; } [$passed, $reason] = $this->compareResults($actual[$key], $expVal, $tolerances, $newPath); if (!$passed) { return [false, $reason]; } } } elseif ($expected !== $actual) { return [false, "{$path}: values do not match"]; } return [true, null]; } private function executeMethod(string $method, string $fixture, array $options) { // This is a stub - replace with actual SDK calls when available return match ($method) { 'extract' => [ 'schema_version' => '1.0', 'metadata' => ['page_count' => 1], 'pages' => [ [ 'page_index' => 0, 'width' => 612, 'height' => 792, 'rotation' => 0, ] ], 'errors' => [] ], 'extract_text' => 'Sample text content', 'extract_markdown' => "# Sample Markdown\n\nContent here", 'hash' => ['hash' => 'abc123', 'fast_hash' => 'def456'], default => null, }; } private function compareVersions(string $v1, string $v2): int { $parts1 = explode('.', $v1); $parts2 = explode('.', $v2); $max = max(count($parts1), count($parts2)); for ($i = 0; $i < $max; $i++) { $n1 = (int)($parts1[$i] ?? 0); $n2 = (int)($parts2[$i] ?? 0); if ($n1 < $n2) { return -1; } if ($n1 > $n2) { return 1; } } return 0; } /** * @return array{id: string, status: string, actual: mixed, expected: mixed, error: string|null, reason: string|null, duration_ms: int} */ private function runTestCase(array $case, string $schemaVersion, string $fixturesBase): array { $start = microtime(true); $id = $case['id']; // Check min_schema_version if (isset($case['min_schema_version'])) { $minVer = $case['min_schema_version']; if ($this->compareVersions($schemaVersion, $minVer) < 0) { return [ 'id' => $id, 'status' => self::STATUS_SKIP, 'reason' => "Schema version {$schemaVersion} < minimum required {$minVer}", 'duration_ms' => (int)((microtime(true) - $start) * 1000), ]; } } $fixture = $case['fixture']; $method = $case['method']; $options = $case['options'] ?? []; $expected = $case['expected'] ?? []; $tolerances = $case['tolerances'] ?? null; $fixturePath = str_starts_with($fixture, 'http') ? $fixture : $fixturesBase . '/' . $fixture; try { $actual = $this->executeMethod($method, $fixturePath, $options); [$passed, $reason] = $this->compareResults($actual, $expected, $tolerances); return [ 'id' => $id, 'status' => $passed ? self::STATUS_PASS : self::STATUS_FAIL, 'actual' => $actual, 'expected' => $expected, 'reason' => $reason, 'duration_ms' => (int)((microtime(true) - $start) * 1000), ]; } catch (\Exception $e) { return [ 'id' => $id, 'status' => self::STATUS_ERROR, 'expected' => $expected, 'error' => $e->getMessage(), 'duration_ms' => (int)((microtime(true) - $start) * 1000), ]; } } /** * @return array{sdk: string, sdk_version: string, suite_version: string, schema_version: string, timestamp: string, results: array, summary: array, environment: array} */ private function runConformance(string $suitePath, string $outputPath): array { echo "pdftract SDK Conformance Runner\n"; echo "SDK: " . self::SDK_NAME . " v" . self::SDK_VERSION . "\n"; echo "Suite: {$suitePath}\n\n"; $suiteContent = file_get_contents($suitePath); if ($suiteContent === false) { throw new \RuntimeException("Failed to read suite from {$suitePath}"); } $suite = json_decode($suiteContent, true, 512, JSON_THROW_ON_ERROR); $suiteVersion = $suite['version']; $schemaVersion = $suite['schema_version']; $cases = $suite['cases']; $fixturesBase = dirname($suitePath) . '/fixtures'; echo "Found " . count($cases) . " test cases\n\n"; $start = microtime(true); $results = []; foreach ($cases as $case) { $result = $this->runTestCase($case, $schemaVersion, $fixturesBase); $results[] = $result; $statusSym = match ($result['status']) { self::STATUS_PASS => 'PASS', self::STATUS_FAIL => 'FAIL', self::STATUS_SKIP => 'SKIP', self::STATUS_ERROR => 'ERROR', }; echo "[{$statusSym}] {$result['id']} ({$result['duration_ms']}ms)\n"; if ($result['status'] === self::STATUS_FAIL || $result['status'] === self::STATUS_ERROR) { if ($result['reason']) { echo " Reason: {$result['reason']}\n"; } if ($result['error']) { echo " Error: {$result['error']}\n"; } } } $durationMs = (int)((microtime(true) - $start) * 1000); $summary = [ 'total' => count($results), 'passed' => count(array_filter($results, fn($r) => $r['status'] === self::STATUS_PASS)), 'failed' => count(array_filter($results, fn($r) => $r['status'] === self::STATUS_FAIL)), 'skipped' => count(array_filter($results, fn($r) => $r['status'] === self::STATUS_SKIP)), 'errors' => count(array_filter($results, fn($r) => $r['status'] === self::STATUS_ERROR)), 'duration_ms' => $durationMs, ]; echo "\nSummary:\n"; echo " Total: {$summary['total']}\n"; echo " Passed: {$summary['passed']}\n"; echo " Failed: {$summary['failed']}\n"; echo " Skipped: {$summary['skipped']}\n"; echo " Errors: {$summary['errors']}\n"; echo " Time: {$summary['duration_ms']}ms\n"; $report = [ 'sdk' => self::SDK_NAME, 'sdk_version' => self::SDK_VERSION, 'suite_version' => $suiteVersion, 'schema_version' => $schemaVersion, 'timestamp' => gmdate('c'), 'results' => $results, 'summary' => $summary, 'environment' => [ 'os' => PHP_OS_FAMILY, 'arch' => php_uname('m'), 'binary_version' => self::SDK_VERSION, 'runtime_version' => PHP_VERSION, ], ]; file_put_contents($outputPath, json_encode($report, JSON_PRETTY_PRINT)); echo "\nReport written to: {$outputPath}\n"; return $report; } public function provideConformanceCases(): iterable { $suitePath = self::SUITE_PATH; $suiteContent = file_get_contents($suitePath); $suite = json_decode($suiteContent, true, 512, JSON_THROW_ON_ERROR); $schemaVersion = $suite['schema_version']; $fixturesBase = dirname($suitePath) . '/fixtures'; foreach ($suite['cases'] as $case) { yield $case['id'] => [$case, $schemaVersion, $fixturesBase]; } } } // CLI entry point if (php_sapi_name() === 'cli' && realpath($argv[0]) === realpath(__FILE__)) { $suiteArg = $argv[1] ?? null; $outputArg = $argv[2] ?? null; $test = new ConformanceTest('testConformance'); $report = $test->runConformance( $suiteArg ?? ConformanceTest::SUITE_PATH, $outputArg ?? 'conformance-report.json' ); exit(($report['summary']['failed'] + $report['summary']['errors']) > 0 ? 1 : 0); }