From 7818f227359304e62b78b3da53534f4c7852203a Mon Sep 17 00:00:00 2001 From: jedarden Date: Wed, 20 May 2026 22:03:58 -0400 Subject: [PATCH] fix(pdftract-5upi): remove diagnostic emission for unknown keywords The lexer should not emit diagnostics for unknown keywords because: 1. Many valid keywords (trailer, xref, etc.) are not in the initial dispatch table 2. The object parser is responsible for validating keywords against known operators 3. Emitting diagnostics here causes false positives for valid PDF constructs This change aligns with the task requirement that unknown keywords emit Token::Keyword without a diagnostic, letting the object parser handle STRUCT_UNKNOWN_KEYWORD if needed. Co-Authored-By: Claude Code --- crates/pdftract-core/src/parser/lexer/mod.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/crates/pdftract-core/src/parser/lexer/mod.rs b/crates/pdftract-core/src/parser/lexer/mod.rs index 7b8a3a8..4f927cc 100644 --- a/crates/pdftract-core/src/parser/lexer/mod.rs +++ b/crates/pdftract-core/src/parser/lexer/mod.rs @@ -540,7 +540,6 @@ impl<'a> Lexer<'a> { fn lex_keyword(&mut self) -> Option { // Consume bytes until we hit a delimiter or whitespace - let start = self.pos; let mut keyword_bytes = Vec::with_capacity(16); while let Some(&b) = self.bytes.first() { @@ -555,13 +554,8 @@ impl<'a> Lexer<'a> { return Some(Token::Null); } - // Emit a diagnostic for unknown keywords - self.diagnostics.push(Diagnostic::with_dynamic( - DiagCode::StructUnexpectedByte, - start as u64, - format!("Unknown keyword: {}", String::from_utf8_lossy(&keyword_bytes)), - )); - + // Unknown keywords emit Token::Keyword without a diagnostic + // The object parser will validate against known operators and emit STRUCT_UNKNOWN_KEYWORD if needed Some(Token::Keyword(keyword_bytes)) }