/* * pdftract SDK Conformance Test Runner (Swift) * * This test runs the shared SDK conformance suite against the Swift SDK. * It loads tests/sdk-conformance/cases.json and executes each test case. * * Run with: swift test --filter ConformanceTests * Or as standalone: swift ConformanceTests.swift */ import Foundation #if canImport(FoundationNetworking) import FoundationNetworking #endif let SUITE_PATH = "tests/sdk-conformance/cases.json" let SDK_NAME = "pdftract-swift" let SDK_VERSION = "0.1.0" enum TestStatus: String, Encodable { case pass = "pass" case fail = "fail" case skip = "skip" case error = "error" } struct TestResult: Encodable { let id: String let status: TestStatus let actual: String? let expected: String? let error: String? let reason: String? let duration_ms: Int64 func toDict() -> [String: Any] { var dict: [String: Any] = [ "id": id, "status": status.rawValue, "duration_ms": duration_ms ] if let actual = actual { dict["actual"] = actual } if let expected = expected { dict["expected"] = expected } if let error = error { dict["error"] = error } if let reason = reason { dict["reason"] = reason } return dict } } struct Summary: Encodable { let total: Int let passed: Int let failed: Int let skipped: Int let errors: Int let duration_ms: Int64 } struct Environment: Encodable { let os: String let arch: String let binary_version: String let runtime_version: String } struct ConformanceReport: Encodable { let sdk: String let sdk_version: String let suite_version: String let schema_version: String let timestamp: String let results: [TestResult] let summary: Summary let environment: Environment } func compareWithTolerance(_ actual: Double, _ expected: Double, _ tolerance: [String: Any]?) -> Bool { guard let tolerance = tolerance else { return abs(actual - expected) < Double.ulpOfOne } if let absTol = tolerance["abs"] as? Double { if abs(actual - expected) <= absTol { return true } } if let relTol = tolerance["rel"] as? Double { let diff = abs(actual - expected) let avg = (actual + expected) / 2.0 if avg > 0.0 && diff / avg <= relTol { return true } } return false } func findTolerance(_ tolerances: [String: Any]?, _ path: String) -> [String: Any]? { guard let tolerances = tolerances else { return nil } if let val = tolerances[path] { return val as? [String: Any] } for (key, val) in tolerances { if key.contains("*") { let pattern = key.replacingOccurrences(of: "*", with: ".*") if let regex = try? NSRegularExpression(pattern: pattern), regex.firstMatch(in: path, range: NSRange(location: 0, length: path.utf16.count)) != nil { return val as? [String: Any] } } } return nil } func compareResults(_ actual: Any, _ expected: Any, _ tolerances: [String: Any]?, _ path: String = "") -> (Bool, String?) { if let expDict = expected as? [String: Any] { if let actNum = actual as? Double { if let min = expDict["min"] as? Double { if actNum < min { return (false, "\(path): value \(actNum) < minimum \(min)") } } if let max = expDict["max"] as? Double { if actNum > max { return (false, "\(path): value \(actNum) > maximum \(max)") } } if let val = expDict["value"] as? Double { let tol = findTolerance(tolerances, path) if !compareWithTolerance(actNum, val, tol) { return (false, "\(path): numeric mismatch") } } } else if let actStr = actual as? String { if let minLen = expDict["min_length"] as? Int { if actStr.count < minLen { return (false, "\(path): string length too short") } } if let contains = expDict["contains"] as? [String] { for substring in contains { if !actStr.contains(substring) { return (false, "\(path): string does not contain '\(substring)'") } } } } else if let actArray = actual as? [Any] { if let min = expDict["min"] as? Int { if actArray.count < min { return (false, "\(path): array length too short") } } if let max = expDict["max"] as? Int { if actArray.count > max { return (false, "\(path): array length too long") } } } else if let actDict = actual as? [String: Any] { for (key, expVal) in expDict { let newPath = path.isEmpty ? key : "\(path).\(key)" guard let actVal = actDict[key] else { return (false, "\(newPath): missing key '\(key)'") } let (passed, reason) = compareResults(actVal, expVal, tolerances, newPath) if !passed { return (false, reason) } } } } else if let expArray = expected as? [Any], let actArray = actual as? [Any] { for (i, expVal) in expArray.enumerated() { let newPath = "\(path)[\(i)]" if i >= actArray.count { return (false, "\(newPath): missing index") } let (passed, reason) = compareResults(actArray[i], expVal, tolerances, newPath) if !passed { return (false, reason) } } } else { // Simple comparison if let actualStr = actual as? String, let expectedStr = expected as? String, actualStr != expectedStr { return (false, "\(path): strings do not match") } } return (true, nil) } func executeMethod(_ method: String, _ fixture: String, _ options: [String: Any]) -> Any { // This is a stub - replace with actual SDK calls when available switch method { case "extract": return [ "schema_version": "1.0", "metadata": ["page_count": 1], "pages": [ [ "page_index": 0, "width": 612, "height": 792, "rotation": 0 ] ], "errors": [] ] as [String: Any] case "extract_text": return "Sample text content" case "extract_markdown": return "# Sample Markdown\n\nContent here" case "hash": return ["hash": "abc123", "fast_hash": "def456"] default: return [:] as [String: Any] } } func compareVersions(_ v1: String, _ v2: String) -> ComparisonResult { let parts1 = v1.split(separator: ".").compactMap { Int($0) } let parts2 = v2.split(separator: ".").compactMap { Int($0) } let maxCount = max(parts1.count, parts2.count) for i in 0.. n2 { return .orderedDescending } } return .orderedSame } func runTestCase(_ case: [String: Any], _ schemaVersion: String, _ fixturesBase: String) -> TestResult { let start = Date() guard let id = case["id"] as? String else { return TestResult( id: "unknown", status: .error, actual: nil, expected: nil, error: "Missing test case ID", reason: nil, duration_ms: 0 ) } // Check min_schema_version if let minVer = case["min_schema_version"] as? String { if compareVersions(schemaVersion, minVer) == .orderedAscending { return TestResult( id: id, status: .skip, actual: nil, expected: nil, error: nil, reason: "Schema version \(schemaVersion) < minimum required \(minVer)", duration_ms: Int64(Date().timeIntervalSince(start) * 1000) ) } } guard let fixture = case["fixture"] as? String, let method = case["method"] as? String else { return TestResult( id: id, status: .error, actual: nil, expected: nil, error: "Missing required fields", reason: nil, duration_ms: 0 ) } let options = case["options"] as? [String: Any] ?? [:] let expected = case["expected"] ?? [:] let tolerances = case["tolerances"] as? [String: Any] let fixturePath: String if fixture.hasPrefix("http") { fixturePath = fixture } else { fixturePath = "\(fixturesBase)/\(fixture)" } do { let actual = executeMethod(method, fixturePath, options) let (passed, reason) = compareResults(actual, expected, tolerances) if passed { return TestResult( id: id, status: .pass, actual: String(describing: actual), expected: String(describing: expected), error: nil, reason: nil, duration_ms: Int64(Date().timeIntervalSince(start) * 1000) ) } else { return TestResult( id: id, status: .fail, actual: String(describing: actual), expected: String(describing: expected), error: nil, reason: reason, duration_ms: Int64(Date().timeIntervalSince(start) * 1000) ) } } catch { return TestResult( id: id, status: .error, actual: nil, expected: String(describing: expected), error: String(describing: error), reason: nil, duration_ms: Int64(Date().timeIntervalSince(start) * 1000) ) } } func runConformance(_ suitePath: String, _ outputPath: String) -> ConformanceReport { print("pdftract SDK Conformance Runner") print("SDK: \(SDK_NAME) v\(SDK_VERSION)") print("Suite: \(suitePath)") print("") guard let suiteData = try? Data(contentsOf: URL(fileURLWithPath: suitePath)), let suite = try? JSONSerialization.jsonObject(with: suiteData) as? [String: Any] else { fatalError("Failed to load suite") } let suiteVersion = suite["version"] as? String ?? "unknown" let schemaVersion = suite["schema_version"] as? String ?? "unknown" let cases = suite["cases"] as? [[String: Any]] ?? [] let fixturesBase = ((suitePath as NSString).deletingLastPathComponent as NSString).appendingPathComponent("fixtures") print("Found \(cases.count) test cases") print("") let start = Date() var results: [TestResult] = [] for testCase in cases { let result = runTestCase(testCase, schemaVersion, fixturesBase) results.append(result) let statusSym: String switch result.status { case .pass: statusSym = "PASS" case .fail: statusSym = "FAIL" case .skip: statusSym = "SKIP" case .error: statusSym = "ERROR" } print("[\(statusSym)] \(result.id) (\(result.duration_ms)ms)") if result.status == .fail || result.status == .error { if let reason = result.reason { print(" Reason: \(reason)") } if let error = result.error { print(" Error: \(error)") } } } let duration_ms = Int64(Date().timeIntervalSince(start) * 1000) let passed = results.filter { $0.status == .pass }.count let failed = results.filter { $0.status == .fail }.count let skipped = results.filter { $0.status == .skip }.count let errors = results.filter { $0.status == .error }.count print("") print("Summary:") print(" Total: \(results.count)") print(" Passed: \(passed)") print(" Failed: \(failed)") print(" Skipped: \(skipped)") print(" Errors: \(errors)") print(" Time: \(duration_ms)ms") let report = ConformanceReport( sdk: SDK_NAME, sdk_version: SDK_VERSION, suite_version: suiteVersion, schema_version: schemaVersion, timestamp: ISO8601DateFormatter().string(from: Date()), results: results, summary: Summary( total: results.count, passed: passed, failed: failed, skipped: skipped, errors: errors, duration_ms: duration_ms ), environment: Environment( os: "macOS", // Runtime detection would go here arch: "arm64", binary_version: SDK_VERSION, runtime_version: "5.9" ) ) if let reportData = try? JSONEncoder().encode(report), let reportJson = String(data: reportData, encoding: .utf8) { try? reportJson.write(toFile: outputPath, atomically: true, encoding: .utf8) print("") print("Report written to: \(outputPath)") } return report } // CLI entry point if CommandLine.argc > 1 { let suiteArg = CommandLine.arguments.count > 1 ? CommandLine.arguments[1] : SUITE_PATH let outputArg = CommandLine.arguments.count > 2 ? CommandLine.arguments[2] : "conformance-report.json" let report = runConformance(suiteArg, outputArg) exit(report.summary.failed == 0 && report.summary.errors == 0 ? 0 : 1) }