/** * pdftract SDK Conformance Test Runner (Node.js / TypeScript) * * This test runs the shared SDK conformance suite against the Node.js SDK. * It loads tests/sdk-conformance/cases.json and executes each test case. * * Run with: vitest test/conformance/conformance.test.ts * Or as standalone: ts-node test/conformance/conformance.test.ts */ import { readFileSync, writeFileSync } from 'fs'; import { join } from 'path'; import { fileURLToPath } from 'url'; const __filename = fileURLToPath(import.meta.url); const __dirname = join(__filename, '..'); const SUITE_PATH = join(__dirname, '..', '..', 'sdk-conformance', 'cases.json'); const SDK_NAME = 'pdftract-node'; const SDK_VERSION = '0.1.0'; enum TestStatus { Pass = 'pass', Fail = 'fail', Skip = 'skip', Error = 'error', } interface TestResult { id: string; status: TestStatus; actual?: any; expected?: any; error?: string; reason?: string; duration_ms: number; } interface ConformanceReport { sdk: string; sdk_version: string; suite_version: string; schema_version: string; timestamp: string; results: TestResult[]; summary: { total: number; passed: number; failed: number; skipped: number; errors: number; duration_ms: number; }; environment: { os: string; arch: string; binary_version: string; runtime_version: string; }; } interface SuiteCase { id: string; fixture: string; method: string; options: Record; expected: any; tolerances?: Record; feature?: string; min_schema_version?: string; skip_reason?: string; } interface Suite { version: string; schema_version: string; cases: SuiteCase[]; } function loadSuite(path: string): Suite { const content = readFileSync(path, 'utf-8'); return JSON.parse(content); } function compareWithTolerance( actual: number, expected: number, tolerance?: { abs?: number; rel?: number } ): boolean { if (!tolerance) { return Math.abs(actual - expected) < Number.EPSILON; } if (tolerance.abs !== undefined) { if (Math.abs(actual - expected) <= tolerance.abs) { return true; } } if (tolerance.rel !== undefined) { const diff = Math.abs(actual - expected); const avg = (actual + expected) / 2.0; if (avg > 0.0 && diff / avg <= tolerance.rel) { return true; } } return false; } function findTolerance( tolerances: Record | undefined, path: string ): { abs?: number; rel?: number } | undefined { if (!tolerances) { return undefined; } if (path in tolerances) { return tolerances[path]; } for (const [key, val] of Object.entries(tolerances)) { if (key.includes('*')) { const pattern = key.replace(/\*/g, '.*'); const regex = new RegExp(pattern); if (regex.test(path)) { return val; } } } return undefined; } function compareResults( actual: any, expected: any, tolerances: Record | undefined, path: string = '' ): { passed: boolean; reason?: string } { if (typeof expected === 'object' && expected !== null && !Array.isArray(expected)) { if ('min' in expected && typeof actual === 'number') { if (actual < expected.min) { return { passed: false, reason: `${path}: value ${actual} < minimum ${expected.min}` }; } } if ('max' in expected && typeof actual === 'number') { if (actual > expected.max) { return { passed: false, reason: `${path}: value ${actual} > maximum ${expected.max}` }; } } if ('value' in expected && typeof actual === 'number') { const tol = findTolerance(tolerances, path); if (!compareWithTolerance(actual, expected.value, tol)) { return { passed: false, reason: `${path}: numeric mismatch` }; } } if ('min_length' in expected && typeof actual === 'string') { if (actual.length < expected.min_length) { return { passed: false, reason: `${path}: string length ${actual.length} < minimum ${expected.min_length}` }; } } if ('contains' in expected && typeof actual === 'string') { for (const substring of expected.contains) { if (!actual.includes(substring)) { return { passed: false, reason: `${path}: string does not contain '${substring}'` }; } } } if ('min' in expected && Array.isArray(actual)) { if (actual.length < expected.min) { return { passed: false, reason: `${path}: array length ${actual.length} < minimum ${expected.min}` }; } } if ('max' in expected && Array.isArray(actual)) { if (actual.length > expected.max) { return { passed: false, reason: `${path}: array length ${actual.length} > maximum ${expected.max}` }; } } // Nested object comparison if (typeof actual === 'object' && actual !== null) { for (const [key, expVal] of Object.entries(expected)) { const newPath = path ? `${path}.${key}` : key; if (!(key in actual)) { return { passed: false, reason: `${newPath}: missing key '${key}'` }; } const result = compareResults(actual[key], expVal, tolerances, newPath); if (!result.passed) { return result; } } } } else if (Array.isArray(expected) && Array.isArray(actual)) { for (let i = 0; i < expected.length; i++) { const newPath = `${path}[${i}]`; if (i >= actual.length) { return { passed: false, reason: `${newPath}: missing index` }; } const result = compareResults(actual[i], expected[i], tolerances, newPath); if (!result.passed) { return result; } } } else { if (actual !== expected) { return { passed: false, reason: `${path}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}` }; } } return { passed: true }; } async function executeMethod( method: string, fixture: string, options: Record ): Promise { // This is a stub - replace with actual SDK calls when available switch (method) { case 'extract': return { schema_version: '1.0', metadata: { page_count: 1 }, pages: [ { page_index: 0, width: 612, height: 792, rotation: 0, }, ], errors: [], }; case 'extract_text': return 'Sample text content'; case 'extract_markdown': return '# Sample Markdown\n\nContent here'; case 'extract_stream': return { output_type: 'iterator', frame_count: 3 }; case 'search': return { output_type: 'iterator', matches: [{ page: 0, text: 'found' }] }; case 'get_metadata': return { metadata: { page_count: 1, title: 'Test', author: 'Test' } }; case 'hash': return { hash: 'abc123', fast_hash: 'def456' }; case 'classify': return { category: 'scientific_paper', confidence: 0.85, tags: ['academic'] }; case 'verify_receipt': return { valid: true }; default: return null; } } async function runTestCase( case: SuiteCase, schemaVersion: string, fixturesBase: string ): Promise { const startTime = Date.now(); // Check min_schema_version if (case.min_schema_version) { const [major, minor] = schemaVersion.split('.').map(Number); const [minMajor, minMinor] = case.min_schema_version.split('.').map(Number); if (major < minMajor || (major === minMajor && minor < minMinor)) { return { id: case.id, status: TestStatus.Skip, reason: `Schema version ${schemaVersion} < minimum required ${case.min_schema_version}`, duration_ms: Date.now() - startTime, }; } } const fixturePath = case.fixture.startsWith('http') ? case.fixture : join(fixturesBase, case.fixture); try { const actual = await executeMethod(case.method, fixturePath, case.options); const { passed, reason } = compareResults(actual, case.expected, case.tolerances); return { id: case.id, status: passed ? TestStatus.Pass : TestStatus.Fail, actual, expected: case.expected, reason, duration_ms: Date.now() - startTime, }; } catch (e) { return { id: case.id, status: TestStatus.Error, expected: case.expected, error: e instanceof Error ? e.message : String(e), duration_ms: Date.now() - startTime, }; } } export async function runConformance( suitePath: string = SUITE_PATH, outputPath: string = 'conformance-report.json' ): Promise { const os = process.platform; const arch = process.arch; const runtimeVersion = `Node.js ${process.version}`; console.log(`pdftract SDK Conformance Runner`); console.log(`SDK: ${SDK_NAME} v${SDK_VERSION}`); console.log(`Suite: ${suitePath}`); console.log(); const suite = loadSuite(suitePath); const fixturesBase = join(suitePath, '..', 'fixtures'); console.log(`Found ${suite.cases.length} test cases`); console.log(); const startTime = Date.now(); const results: TestResult[] = []; for (const case_ of suite.cases) { const result = await runTestCase(case_, suite.schema_version, fixturesBase); const statusSym = { [TestStatus.Pass]: 'PASS', [TestStatus.Fail]: 'FAIL', [TestStatus.Skip]: 'SKIP', [TestStatus.Error]: 'ERROR', }[result.status]; console.log(`[${statusSym}] ${result.id} (${result.duration_ms}ms)`); if (result.status === TestStatus.Fail || result.status === TestStatus.Error) { if (result.reason) { console.log(` Reason: ${result.reason}`); } if (result.error) { console.log(` Error: ${result.error}`); } } results.push(result); } const duration_ms = Date.now() - startTime; const summary = { total: results.length, passed: results.filter((r) => r.status === TestStatus.Pass).length, failed: results.filter((r) => r.status === TestStatus.Fail).length, skipped: results.filter((r) => r.status === TestStatus.Skip).length, errors: results.filter((r) => r.status === TestStatus.Error).length, duration_ms, }; console.log(); console.log('Summary:'); console.log(` Total: ${summary.total}`); console.log(` Passed: ${summary.passed}`); console.log(` Failed: ${summary.failed}`); console.log(` Skipped: ${summary.skipped}`); console.log(` Errors: ${summary.errors}`); console.log(` Time: ${summary.duration_ms}ms`); const report: ConformanceReport = { sdk: SDK_NAME, sdk_version: SDK_VERSION, suite_version: suite.version, schema_version: suite.schema_version, timestamp: new Date().toISOString(), results, summary, environment: { os, arch, binary_version: SDK_VERSION, runtime_version: runtimeVersion, }, }; writeFileSync(outputPath, JSON.stringify(report, null, 2)); console.log(); console.log(`Report written to: ${outputPath}`); return report; } // Vitest entry point export async function testConformanceSuite() { const report = await runConformance(); if (report.summary.failed > 0) { throw new Error(`${report.summary.failed} tests failed`); } if (report.summary.errors > 0) { throw new Error(`${report.summary.errors} tests errored`); } } // CLI entry point if (import.meta.url === `file://${process.argv[1]}`) { const suiteArg = process.argv[2]; const outputArg = process.argv[3]; runConformance(suiteArg, outputArg).then((report) => { process.exit(report.summary.failed === 0 && report.summary.errors === 0 ? 0 : 1); }); }