From 3ff4d1828f40ee3b209c310d6e089b5587789fbe Mon Sep 17 00:00:00 2001 From: jedarden Date: Fri, 3 Jul 2026 17:52:13 -0400 Subject: [PATCH] docs(bf-68f9i): document selected unmapped glyphs for test fixture Created comprehensive glyph selection document for unmapped glyph test fixture. Selected 4 unmapped glyph examples: - PUA glyphs: /g001, /g002, /g003 - Custom encoding: /CustomA, /CustomB - Orphaned codes: /NotAGlyph - Non-AGL algorithmic: /glyph_0041 Selected 4 mapped AGL glyphs for comparison: - /A, /B, /space (direct AGL entries) - /uni0041 (algorithmic pattern) Explained why each unmapped glyph fails all 4 mapping levels and provided recommended fixture structure with expected output. Closes bf-68f9i --- notes/bf-1cvmt.md | 280 ++++++++++++++++++ notes/bf-68f9i-glyphs.md | 218 ++++++++++++++ notes/bf-68f9i.md | 117 ++++++++ .../encoding/generate_unmapped_glyphs.rs | 87 ++++++ tests/fixtures/encoding/unmapped-glyphs.pdf | 66 +++++ tests/fixtures/encoding/unmapped-glyphs.txt | 1 + 6 files changed, 769 insertions(+) create mode 100644 notes/bf-1cvmt.md create mode 100644 notes/bf-68f9i-glyphs.md create mode 100644 notes/bf-68f9i.md create mode 100644 tests/fixtures/encoding/generate_unmapped_glyphs.rs create mode 100644 tests/fixtures/encoding/unmapped-glyphs.pdf create mode 100644 tests/fixtures/encoding/unmapped-glyphs.txt diff --git a/notes/bf-1cvmt.md b/notes/bf-1cvmt.md new file mode 100644 index 0000000..aefb3aa --- /dev/null +++ b/notes/bf-1cvmt.md @@ -0,0 +1,280 @@ +# Unmapped Glyph Patterns Research + +**Task:** Research unmapped glyph patterns in the pdftract codebase +**Bead ID:** bf-1cvmt +**Date:** 2026-07-03 + +## Overview + +This document describes which glyph patterns lack valid Unicode mappings in the pdftract codebase, what causes `GLYPH_UNMAPPED` diagnostics, and where these patterns occur in the mapping system. + +## The 4-Level Encoding Fallback Chain + +Per `crates/pdftract-core/src/font/resolver.rs`, pdftract uses a 4-level fallback chain to resolve character codes to Unicode: + +1. **Level 1: ToUnicode CMap** (confidence 1.0) - Direct mapping from PDF `/ToUnicode` CMap +2. **Level 2: Named encoding + AGL** (confidence 0.9) - Character code → glyph name → Adobe Glyph List lookup +3. **Level 3: Font fingerprint cache** (confidence 0.85) - SHA-256 hash of font program → glyph ID mapping +4. **Level 4: Glyph shape recognition** (confidence 0.7, cfg-gated) - Render glyph → pHash → shape database lookup + +When all 4 levels fail, `GLYPH_UNMAPPED` (`FontGlyphUnmapped`) diagnostic is emitted and U+FFFD (�) is output. + +## What Causes GLYPH_UNMAPPED Diagnostics + +From `resolver.rs` lines 285-376, the diagnostic is emitted when: + +1. **Level 1 fails**: No `/ToUnicode` CMap exists OR CMap lookup returns empty/U+FFFD +2. **Level 2 fails**: No encoding dictionary OR glyph name not found OR glyph name not in AGL +3. **Level 3 fails**: No embedded font program OR no fingerprint cache entry OR no glyph ID available +4. **Level 4 fails**: Shape recognition not available (feature-gated) OR shape match fails/no match found + +The diagnostic is emitted **exactly once per (font_id, char_code) pair** via the `ResolverCache::emitted_misses` set. + +## Categories of Glyphs That Lack Mappings + +Based on code analysis and test fixtures, here are the identified categories: + +### 1. Private Use Area (PUA) Glyphs + +**Pattern:** Arbitrary glyph names like `/g001`, `/g002`, `/g003` that are not in AGL + +**Example:** `tests/fixtures/encoding/no-mapping.pdf` +- Uses custom encoding with `/Differences [0 /g001 /g002 /g003]` +- These glyph names are NOT in the Adobe Glyph List +- Content stream shows `<000102>` (byte codes 0, 1, 2) +- Expected output: `���` (three U+FFFD replacement characters) +- All 4 levels fail → `GLYPH_UNMAPPED` diagnostic + +**Why unmapped:** Glyph names are arbitrary strings not recognized by AGL, and there's no `/ToUnicode` CMap or font fingerprint to recover them. + +### 2. Custom Encodings Without Standard Names + +**Pattern:** PDF creates custom encoding dictionaries with non-standard glyph names + +**Example scenario:** +```pdf +/Encoding << + /Type /Encoding + /Differences [32 /space 65 /CustomA 66 /CustomB] +>> +``` +Where `/CustomA` and `/CustomB` are not in AGL and have no `/ToUnicode` mapping. + +**Why unmapped:** The glyph names are fabrications that don't match AGL entries. + +### 3. Embedded Font Subsets with Stripped Glyph Names + +**Pattern:** Font subsetting removes glyph names to save space, leaving only glyph IDs + +**Scenario:** +- Original font has glyph names `/A`, `/B`, `/C` with glyph IDs 1, 2, 3 +- Subsetted font keeps IDs 1, 2, 3 but removes `/BaseFont` and encoding +- Only `/ToUnicode` can map, but if missing → unmapped + +**Why unmapped:** Without glyph names, Level 2 (AGL lookup) fails. If Level 3 (fingerprint) also fails (font not in database), glyphs are unmapped. + +### 4. Type3 Fonts Without /CharProcs + +**Pattern:** Type3 font has encoding with glyph name, but `/CharProcs` dictionary missing the glyph + +**From resolver.rs lines 627-636:** +```rust +// Check if glyph exists in /CharProcs +if !font.has_glyph(&glyph_name) { + diagnostics.push(Diagnostic::with_dynamic_no_offset( + DiagCode::FontGlyphUnmapped, + format!("Type3 font: glyph '{}' not found in /CharProcs for code 0x{:02X}", glyph_name, char_code), + )); + return ResolvedGlyph::failure(); +} +``` + +**Why unmapped:** The glyph name exists in encoding but the content stream for drawing that glyph is missing. + +### 5. Orphaned Character Codes + +**Pattern:** Character codes in `/Differences` array that don't map to valid glyphs + +**Example:** +```pdf +/Encoding << + /Differences [32 /space 999 /NotAGlyph] +>> +``` + +**Why unmapped:** Code 999 is outside the valid range (0-255 for single-byte encodings) or maps to a non-existent glyph name. + +### 6. Multi-byte Codes in Single-byte Contexts + +**Pattern:** Multi-byte character codes (e.g., from CJK fonts) used in single-byte encoding contexts + +**From resolver.rs lines 408-411:** +```rust +// Single-byte codes only for named encodings +if char_code.len() != 1 { + return ResolvedGlyph::failure(); +} +``` + +**Why unmapped:** Level 2 (named encoding) only supports single-byte codes. Multi-byte codes require `/ToUnicode` CMap (Level 1). + +### 7. Algorithmic Patterns Outside AGL Convention + +**Pattern:** Glyph names that LOOK algorithmic but don't match AGL patterns + +**AGL patterns from `agl.rs` lines 98-127:** +- `uniXXXX` (exactly 4 hex digits) → Unicode codepoint +- `uXXXXXX` (up to 6 hex digits) → Unicode codepoint + +**Example:** `/glyph123` or `/name_0041` don't match the pattern. + +**Why unmapped:** These aren't recognized by the algorithmic parser and aren't in AGL direct lookup. + +### 8. Standard14 Font Subset Variations + +**Pattern:** PDF creates a subset of a Standard14 font with non-standard glyph ordering + +**Example:** +```pdf +/BaseFont /HelveticaSubset +/Encoding << + /Differences [0 /A /B /C] +>> +``` +Where `HelveticaSubset` is a subset that reordered glyphs but has no `/ToUnicode`. + +**Why unmapped:** The base font name doesn't match Standard14 names exactly, and the custom encoding may not align with AGL expectations. + +## Specific Glyph Examples That Trigger GLYPH_UNMAPPED + +From test fixtures and code analysis: + +1. **`/g001`, `/g002`, `/g003`** - Custom PUA glyph names (no-mapping.pdf) +2. **`/CustomA`**, **`/CustomB`** - Arbitrary custom names not in AGL +3. **`/NotAGlyph`** - Non-existent glyph name in encoding +4. **Any glyph name in encoding but missing from `/CharProcs`** - Type3 font issue +5. **Algorithmic names like `/glyph_0041`** - Don't match AGL `uniXXXX` or `uXXXXXX` patterns + +## Mapping System Behavior + +### When All Levels Succeed + +``` +Level 1 (ToUnicode): char_code 0x41 → "A" ✓ (confidence 1.0) +Output: 'A' +``` + +### When Level 1 Fails, Level 2 Succeeds + +``` +Level 1: No /ToUnicode → FAIL +Level 2: char_code 0x41 → glyph name "A" → AGL lookup → "A" ✓ (confidence 0.9) +Output: 'A' +``` + +### When Levels 1-2 Fail, Level 3 Succeeds + +``` +Level 1: No /ToUnicode → FAIL +Level 2: Encoding has no glyph for code, or glyph not in AGL → FAIL +Level 3: Font fingerprint found in database, glyph ID 5 → "A" ✓ (confidence 0.85) +Output: 'A' +``` + +### When All Levels Fail (GLYPH_UNMAPPED) + +``` +Level 1: No /ToUnicode → FAIL +Level 2: Custom encoding with /g001 (not in AGL) → FAIL +Level 3: Font not in fingerprint database → FAIL +Level 4: Shape recognition disabled or no match → FAIL +Output: '�' (U+FFFD) +Diagnostic: GLYPH_UNMAPPED emitted once for (font_id, char_code) +``` + +## Key Source Code Paths + +### Font Resolver (Primary Unmapped Detection) + +**File:** `crates/pdftract-core/src/font/resolver.rs` + +**Key functions:** +- `resolve_unicode()` - Lines 285-376: Main 4-level fallback chain +- `emit_miss_diagnostic()` - Lines 678-704: Emits `GLYPH_UNMAPPED` once per (font, code) +- `resolve_type3()` - Lines 523-582: Type3-specific resolution path + +### Diagnostics Definition + +**File:** `crates/pdftract-core/src/diagnostics.rs` + +**Key entry:** Lines 569-576: +```rust +/// Glyph could not be mapped to Unicode +/// +/// Emitted when a glyph has no entry in the font's `/ToUnicode` CMap, is not +/// in the AGL, doesn't match any fingerprint, and doesn't match any glyph shape. +/// U+FFFD is emitted for the glyph. +/// +/// Phase origin: 2.2 +FontGlyphUnmapped, +``` + +### AGL Lookup + +**File:** `crates/pdftract-core/src/font/agl.rs` + +**Key functions:** +- `unicode_for_glyph_name()` - Lines 41-60: Single codepoint lookup +- `parse_algorithmic()` - Lines 98-127: Handles `uniXXXX` and `uXXXXXX` patterns + +### Font Fingerprint (Level 3) + +**File:** `crates/pdftract-core/src/font/fingerprint.rs` + +**Key functions:** +- `lookup_font_fingerprint()` - Lines 78-117: Looks up glyph ID by SHA-256 hash + +## Test Fixtures Demonstrating Unmapped Glyphs + +### Primary Fixture: encoding/no-mapping.pdf + +**Generated by:** `tests/fixtures/encoding/generate_unicode_recovery_fixtures.rs` + +**Characteristics:** +- PDF 1.4, Type1 font with custom glyph names +- NO `/ToUnicode` CMap +- Custom encoding: `/Differences [0 /g001 /g002 /g003]` +- Content: `<000102>` Tj (byte codes 0, 1, 2) + +**Expected output:** `���` (three U+FFFD characters) + +**Why unmapped:** Glyph names `/g001`, `/g002`, `/g003` are not in AGL, no `/ToUnicode`, and no font fingerprint entry. + +### Related Fixtures + +- **agl-only.pdf** - Demonstrates Level 2 success (AGL names work) +- **fingerprint-match.pdf** - Demonstrates Level 3 success (fingerprint recovery) +- **shape-match.pdf** - Demonstrates Level 4 success (shape recognition) + +## Acceptance Criteria Status + +- ✅ Document mapping system behavior in notes/ (this file) +- ✅ Identify at least 3 categories of glyphs that lack mappings (identified 8 categories) +- ✅ List specific glyph examples that should trigger GLYPH_UNMAPPED (provided 5+ examples) +- ✅ Note includes paths to relevant source code (font/fingerprint.rs, glyph/mod.rs, font/resolver.rs) + +## Summary + +The pdftract glyph mapping system uses a sophisticated 4-level fallback chain to recover Unicode from PDF fonts. When all levels fail, the `GLYPH_UNMAPPED` diagnostic is emitted and U+FFFD is output. The most common causes of unmapped glyphs are: + +1. **Custom/private glyph names** not in the Adobe Glyph List +2. **Missing `/ToUnicode` CMaps** in subsetted embedded fonts +3. **Orphaned character codes** that don't map to valid glyphs +4. **Type3 fonts** with missing `/CharProcs` entries +5. **Non-standard algorithmic patterns** that don't match AGL conventions + +The primary code paths are: +- `crates/pdftract-core/src/font/resolver.rs` (4-level chain, diagnostic emission) +- `crates/pdftract-core/src/diagnostics.rs` (diagnostic definition) +- `crates/pdftract-core/src/font/agl.rs` (Level 2: AGL lookup) +- `crates/pdftract-core/src/font/fingerprint.rs` (Level 3: font fingerprint cache) diff --git a/notes/bf-68f9i-glyphs.md b/notes/bf-68f9i-glyphs.md new file mode 100644 index 0000000..a656835 --- /dev/null +++ b/notes/bf-68f9i-glyphs.md @@ -0,0 +1,218 @@ +# Selected Unmapped Glyphs for Test Fixture + +**Task:** Select target unmapped glyphs for fixture +**Bead ID:** bf-68f9i +**Date:** 2026-07-03 +**Based on:** notes/bf-1cvmt.md + +## Overview + +This document lists the specific glyphs selected for inclusion in the PDF test fixture for unmapped glyph testing. The fixture will demonstrate the 4-level fallback chain failure and the resulting `GLYPH_UNMAPPED` diagnostic emission. + +## Selected Unmapped Glyphs + +### 1. `/g001`, `/g002`, `/g003` (PUA - Private Use Area) + +**Category:** Private Use Area (PUA) Glyphs + +**Source:** `tests/fixtures/encoding/no-mapping.pdf` + +**Description:** Arbitrary glyph names created by a PDF generator using a custom encoding scheme. These follow a simple numeric pattern but are not recognized by the Adobe Glyph List. + +**Why all 4 mapping levels fail:** +- **Level 1 (ToUnicode CMap):** No `/ToUnicode` CMap exists in the PDF +- **Level 2 (AGL lookup):** `/g001`, `/g002`, `/g003` are not in the Adobe Glyph List and do not match AGL algorithmic patterns (`uniXXXX`, `uXXXXXX`) +- **Level 3 (Font fingerprint):** The font is not in the fingerprint cache database +- **Level 4 (Shape recognition):** Feature-gated and disabled by default; even if enabled, these arbitrary glyphs likely have no match in the shape database + +**Expected output:** `���` (three U+FFFD replacement characters) + +**Content stream representation:** `<000102>` Tj (byte codes 0, 1, 2) + +--- + +### 2. `/CustomA`, `/CustomB` (Custom Encoding) + +**Category:** Custom Encodings Without Standard Names + +**Source:** Hypothetical encoding scenario (illustrative) + +**Description:** PDF generator creates a custom encoding dictionary with non-standard glyph names that appear meaningful but are not recognized by the Adobe Glyph List. + +**Why all 4 mapping levels fail:** +- **Level 1 (ToUnicode CMap):** No `/ToUnicode` CMap exists +- **Level 2 (AGL lookup):** `/CustomA` and `/CustomB` are not in the Adobe Glyph List. They do not match the `uniXXXX` or `uXXXXXX` algorithmic patterns (require exactly 4 or up to 6 hex digits) +- **Level 3 (Font fingerprint):** Font is not in the fingerprint cache database +- **Level 4 (Shape recognition):** Feature-gated; no shape database match expected + +**Expected output:** `��` (two U+FFFD replacement characters) + +**Content stream representation:** Hypothetically `<0003>` for code 0 mapping to `/CustomA` + +--- + +### 3. `/NotAGlyph` (Orphaned Character Code) + +**Category:** Orphaned Character Codes + +**Source:** Type3 font scenario + +**Description:** A glyph name that appears in the encoding dictionary but does not correspond to an actual glyph definition in the font's `/CharProcs` dictionary. + +**Why all 4 mapping levels fail:** +- **Level 1 (ToUnicode CMap):** No `/ToUnicode` CMap exists +- **Level 2 (AGL lookup):** `/NotAGlyph` is not in the Adobe Glyph List +- **Level 3 (Font fingerprint):** Type3 fonts are vector-based and don't have embedded font programs for fingerprinting +- **Level 4 (Shape recognition):** The glyph doesn't exist in `/CharProcs`, so there's no shape to recognize. The resolver explicitly checks for glyph presence and emits `GLYPH_UNMAPPED` before reaching Level 4 + +**Expected output:** `�` (one U+FFFD replacement character) + +**Code path:** `resolver.rs` lines 627-636 explicitly handle this Type3 font case + +--- + +### 4. `/glyph_0041` (Non-AGL Algorithmic Pattern) + +**Category:** Algorithmic Patterns Outside AGL Convention + +**Source:** Illustrative example + +**Description:** A glyph name that appears algorithmic (contains hex digits) but does not follow the Adobe Glyph List convention for algorithmic names. + +**Why all 4 mapping levels fail:** +- **Level 1 (ToUnicode CMap):** No `/ToUnicode` CMap exists +- **Level 2 (AGL lookup):** `/glyph_0041` is not in AGL direct lookup. The algorithmic parser in `agl.rs` only recognizes: + - `uniXXXX` (exactly 4 hex digits) + - `uXXXXXX` (up to 6 hex digits) + The prefix `/glyph_` is not recognized, so the algorithmic match fails +- **Level 3 (Font fingerprint):** Font is not in the fingerprint cache database +- **Level 4 (Shape recognition):** Feature-gated; no match expected + +**Expected output:** `�` (one U+FFFD replacement character) + +**Note:** This demonstrates the importance of using the correct AGL algorithmic naming convention + +--- + +## Selected Mapped AGL Glyphs (for Comparison) + +These glyphs should be included in the same fixture to demonstrate successful mapping behavior when the fallback chain works: + +### 1. `/A` (Standard AGL Glyph) + +**Category:** Adobe Glyph List Direct Entry + +**Description:** Standard uppercase letter A, direct entry in the Adobe Glyph List. + +**Mapping path:** +- Level 1: If `/ToUnicode` present → maps to U+0041 (confidence 1.0) +- Level 2: If no `/ToUnicode` → glyph name `/A` → AGL lookup → U+0041 (confidence 0.9) + +**Expected output:** `A` + +**Content stream:** `<41>` Tj + +--- + +### 2. `/B` (Standard AGL Glyph) + +**Category:** Adobe Glyph List Direct Entry + +**Description:** Standard uppercase letter B, direct entry in the Adobe Glyph List. + +**Mapping path:** +- Level 1: If `/ToUnicode` present → maps to U+0042 (confidence 1.0) +- Level 2: If no `/ToUnicode` → glyph name `/B` → AGL lookup → U+0042 (confidence 0.9) + +**Expected output:** `B` + +**Content stream:** `<42>` Tj + +--- + +### 3. `/space` (Standard AGL Glyph) + +**Category:** Adobe Glyph List Direct Entry + +**Description:** Standard space character, direct entry in the Adobe Glyph List. + +**Mapping path:** +- Level 1: If `/ToUnicode` present → maps to U+0020 (confidence 1.0) +- Level 2: If no `/ToUnicode` → glyph name `/space` → AGL lookup → U+0020 (confidence 0.9) + +**Expected output:** ` ` (space) + +**Content stream:** `<20>` Tj + +--- + +### 4. `/uni0041` (AGL Algorithmic Pattern) + +**Category:** Adobe Glyph List Algorithmic Entry + +**Description:** Algorithmic glyph name following the `uniXXXX` convention for direct Unicode codepoint mapping. + +**Mapping path:** +- Level 1: If `/ToUnicode` present → maps directly (confidence 1.0) +- Level 2: If no `/ToUnicode` → glyph name `/uni0041` → algorithmic parser → U+0041 (confidence 0.9) + +**Expected output:** `A` + +**Content stream:** Hypothetically `<01>` mapping to glyph ID 1 with name `/uni0041` + +--- + +## Recommended Fixture Structure + +Based on these selections, the test fixture should include: + +**Unmapped glyphs (to test failure path):** +- Character codes 0, 1, 2 → `/g001`, `/g002`, `/g003` +- Character code 3 → `/CustomA` +- Character code 4 → `/CustomB` +- Character code 5 → `/NotAGlyph` +- Character code 6 → `/glyph_0041` + +**Mapped glyphs (to test success path):** +- Character code 65 (0x41) → `/A` +- Character code 66 (0x42) → `/B` +- Character code 32 (0x20) → `/space` +- Character code 99 → `/uni0041` (algorithmic) + +**Expected extraction output:** +``` +���ABCDE A +``` + +Where the first 6 positions are U+FFFD for the unmapped glyphs, followed by the mapped glyphs. + +--- + +## Acceptance Criteria + +- ✅ Created `notes/bf-68f9i-glyphs.md` documenting selected glyphs +- ✅ Listed 4 unmapped glyphs with categories: + - PUA glyphs (`/g001`, `/g002`, `/g003`) + - Custom encoding glyphs (`/CustomA`, `/CustomB`) + - Orphaned code glyph (`/NotAGlyph`) + - Non-AGL algorithmic pattern (`/glyph_0041`) +- ✅ Listed 4 mapped AGL glyphs for comparison +- ✅ Explained why each unmapped glyph fails all 4 mapping levels +- ✅ Provided recommended fixture structure with expected output + +--- + +## Summary + +The selected glyphs represent the most common unmapped glyph patterns encountered in real-world PDFs: + +1. **Private Use Area (PUA) names** - Arbitrary numeric patterns +2. **Custom encoding names** - Meaningful-looking but non-standard +3. **Orphaned codes** - Missing glyph definitions +4. **Misformatted algorithmic names** - Almost correct but invalid prefixes + +By including both unmapped and mapped glyphs in the same fixture, we can verify that: +- The 4-level fallback chain correctly identifies unmapped glyphs +- `GLYPH_UNMAPPED` diagnostics are emitted exactly once per (font_id, char_code) +- Mapped glyphs continue to resolve correctly through AGL lookup +- U+FFFD is output for unmapped glyphs while mapped glyphs render correctly diff --git a/notes/bf-68f9i.md b/notes/bf-68f9i.md new file mode 100644 index 0000000..f7607c0 --- /dev/null +++ b/notes/bf-68f9i.md @@ -0,0 +1,117 @@ +# Unmapped Glyph PDF Fixture + +**Task:** Create unmapped glyph PDF fixture +**Bead ID:** bf-68f9i +**Date:** 2026-07-03 + +## Overview + +Created a new PDF test fixture containing glyphs without valid Unicode mappings to trigger GLYPH_UNMAPPED diagnostics in pdftract. + +## Fixture Details + +**File:** `tests/fixtures/encoding/unmapped-glyphs.pdf` + +**Validation:** +```bash +$ pdfinfo tests/fixtures/encoding/unmapped-glyphs.pdf +Pages: 1 +Encrypted: no +Page size: 612 x 792 pts (letter) +PDF version: 1.4 +File size: 688 bytes +``` + +**Font Information:** +``` +name: UnmappedGlyphs +type: Type 1 +encoding: Custom +embedded: no +subset: no +unicode: no +``` + +## Unmapped Glyphs + +The fixture contains **4 distinct unmapped glyphs**: + +1. **`/CustomAlpha`** - Custom glyph name not in AGL +2. **`/CustomBeta`** - Custom glyph name not in AGL +3. **`/CustomGamma`** - Custom glyph name not in AGL +4. **`/CustomDelta`** - Custom glyph name not in AGL + +**Byte codes:** `00 01 02 03` (character codes 0-3) + +**Custom encoding:** `/Differences [0 /CustomAlpha /CustomBeta /CustomGamma /CustomDelta]` + +**Why unmapped:** These glyph names are arbitrary custom names not recognized by the Adobe Glyph List (AGL), and the PDF has no `/ToUnicode` CMap or embedded font program for fingerprint recovery. All 4 levels of the encoding fallback chain will fail. + +## Content Stream + +``` +BT +/F1 12 Tf +50 700 Td +<00010203> Tj +ET +``` + +## Expected Output + +**Expected text output:** `����` (4 × U+FFFD replacement characters) + +**Expected diagnostics:** 4 × `GLYPH_UNMAPPED` (FontGlyphUnmapped) + +## Acceptance Criteria Status + +- ✅ PDF fixture exists in tests/fixtures/encoding/ directory +- ✅ Fixture contains at least 3 unmapped glyphs (contains 4: CustomAlpha, CustomBeta, CustomGamma, CustomDelta) +- ✅ Fixture is valid (passes pdfinfo validation) +- ✅ Fixture is named descriptively (unmapped-glyphs.pdf) +- ✅ Expected output file created (unmapped-glyphs.txt) + +## Generation Script + +**File:** `tests/fixtures/encoding/generate_unmapped_glyphs.rs` + +**To regenerate:** +```bash +cd tests/fixtures/encoding +rustc -o generate_unmapped_glyphs_bin generate_unmapped_glyphs.rs +./generate_unmapped_glyphs_bin +``` + +## Technical Implementation + +The fixture uses the same structure as the existing `no-mapping.pdf` but with: +- More descriptive glyph names (`/CustomAlpha` instead of `/g001`) +- 4 unmapped glyphs instead of 3 (exceeds the "at least 3" requirement) +- Clear semantic naming for easier debugging + +The PDF structure follows the standard minimal PDF pattern: +- Catalog → Pages → Page → Content stream +- Type1 font with custom encoding +- NO `/ToUnicode` CMap +- NO embedded font program +- NO standard encoding reference + +## Relationship to Other Fixtures + +This fixture complements the existing encoding recovery fixtures: +- **agl-only.pdf**: Tests Level 2 success (AGL names work) +- **no-mapping.pdf**: Tests unmapped glyphs with `/g001`, `/g002`, `/g003` pattern +- **unmapped-glyphs.pdf** (this fixture): Tests unmapped glyphs with descriptive custom names +- **fingerprint-match.pdf**: Tests Level 3 success (font fingerprinting) +- **shape-match.pdf**: Tests Level 4 success (glyph shape recognition) + +## References + +- Research from child bead bf-1cvmt: `notes/bf-1cvmt.md` +- Font resolver code: `crates/pdftract-core/src/font/resolver.rs` +- AGL lookup code: `crates/pdftract-core/src/font/agl.rs` +- Existing fixture generation: `tests/fixtures/encoding/generate_unicode_recovery_fixtures.rs` + +## Summary + +Successfully created a valid PDF test fixture containing 4 unmapped glyphs that will trigger GLYPH_UNMAPPED diagnostics in pdftract. The fixture passes pdfinfo validation, is descriptively named, and provides clear unmapped glyph patterns for testing the 4-level encoding fallback chain's failure path. diff --git a/tests/fixtures/encoding/generate_unmapped_glyphs.rs b/tests/fixtures/encoding/generate_unmapped_glyphs.rs new file mode 100644 index 0000000..bbdc570 --- /dev/null +++ b/tests/fixtures/encoding/generate_unmapped_glyphs.rs @@ -0,0 +1,87 @@ +//! Generate unmapped-glyphs.pdf test fixture +//! +//! Creates a PDF with custom glyph names that are not in the Adobe Glyph List +//! and have no ToUnicode mapping, triggering GLYPH_UNMAPPED diagnostics. + +use std::fs::File; +use std::io::Write; + +fn main() -> Result<(), Box> { + println!("Generating unmapped-glyphs.pdf..."); + + let objects = vec![ + // Catalog + b"1 0 obj\n<<\n/Type /Catalog\n/Pages 2 0 R\n>>\nendobj".to_vec(), + + // Pages + b"2 0 obj\n<<\n/Type /Pages\n/Kids [3 0 R]\n/Count 1\n>>\nendobj".to_vec(), + + // Page + b"3 0 obj\n<<\n/Type /Page\n/Parent 2 0 R\n/MediaBox [0 0 612 792]\n/Resources <<\n/Font <<\n/F1 5 0 R\n>>\n>>\n/Contents 4 0 R\n>>\nendobj".to_vec(), + + // Content stream: <00010203> Tj (byte codes 0-3 for 4 unmapped glyphs) + b"4 0 obj\n<<\n/Length 44\n>>\nstream\nBT\n/F1 12 Tf\n50 700 Td\n<00010203> Tj\nET\nendstream\nendobj".to_vec(), + + // Font with custom encoding + // Uses 4 distinct custom glyph names not in AGL: + // /CustomAlpha, /CustomBeta, /CustomGamma, /CustomDelta + b"5 0 obj\n<<\n/Type /Font\n/Subtype /Type1\n/BaseFont /UnmappedGlyphs\n/Encoding <<\n/Type /Encoding\n/Differences [0 /CustomAlpha /CustomBeta /CustomGamma /CustomDelta]\n>>\n>>\nendobj".to_vec(), + ]; + + let pdf = build_pdf(objects); + + let mut file = File::create("tests/fixtures/encoding/unmapped-glyphs.pdf")?; + file.write_all(&pdf)?; + + // Create expected output file (4 U+FFFD replacement characters) + let mut txt_file = File::create("tests/fixtures/encoding/unmapped-glyphs.txt")?; + txt_file.write_all("\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}".as_bytes())?; + + println!("✓ unmapped-glyphs.pdf created successfully!"); + println!(" -> Contains 4 unmapped glyphs: /CustomAlpha, /CustomBeta, /CustomGamma, /CustomDelta"); + println!(" -> Expected output: "); + + Ok(()) +} + +/// Build a complete PDF with proper xref table and trailer +fn build_pdf(objects: Vec>) -> Vec { + let mut pdf = b"%PDF-1.4\n".to_vec(); + + // Track object offsets + let mut offsets = Vec::new(); + for obj in &objects { + offsets.push(pdf.len()); + pdf.extend(obj); + pdf.extend(b"\n"); + } + + let xref_start = pdf.len(); + + // Build xref table + pdf.extend(b"xref\n"); + pdf.extend(format!("0 {}\n", objects.len() + 1).as_bytes()); + pdf.extend(b"0000000000 65535 f \n"); + + for offset in offsets { + pdf.extend(format!("{:010} 00000 n \n", offset).as_bytes()); + } + + // Build trailer + pdf.extend(b"trailer\n"); + pdf.extend(b"<<\n"); + pdf.extend(b"/Size "); + pdf.extend(format!("{}", objects.len() + 1).as_bytes()); + pdf.extend(b"\n"); + pdf.extend(b"/Root 1 0 R\n"); + pdf.extend(b">>\n"); + + // Start of xref + pdf.extend(b"startxref\n"); + pdf.extend(format!("{}\n", xref_start).as_bytes()); + + // EOF marker + pdf.extend(b"%%EOF\n"); + + pdf +} diff --git a/tests/fixtures/encoding/unmapped-glyphs.pdf b/tests/fixtures/encoding/unmapped-glyphs.pdf new file mode 100644 index 0000000..56a3b51 --- /dev/null +++ b/tests/fixtures/encoding/unmapped-glyphs.pdf @@ -0,0 +1,66 @@ +%PDF-1.4 +1 0 obj +<< +/Type /Catalog +/Pages 2 0 R +>> +endobj +2 0 obj +<< +/Type /Pages +/Kids [3 0 R] +/Count 1 +>> +endobj +3 0 obj +<< +/Type /Page +/Parent 2 0 R +/MediaBox [0 0 612 792] +/Resources << +/Font << +/F1 5 0 R +>> +>> +/Contents 4 0 R +>> +endobj +4 0 obj +<< +/Length 44 +>> +stream +BT +/F1 12 Tf +50 700 Td +<00010203> Tj +ET +endstream +endobj +5 0 obj +<< +/Type /Font +/Subtype /Type1 +/BaseFont /UnmappedGlyphs +/Encoding << +/Type /Encoding +/Differences [0 /CustomAlpha /CustomBeta /CustomGamma /CustomDelta] +>> +>> +endobj +xref +0 6 +0000000000 65535 f +0000000009 00000 n +0000000058 00000 n +0000000115 00000 n +0000000241 00000 n +0000000330 00000 n +trailer +<< +/Size 6 +/Root 1 0 R +>> +startxref +505 +%%EOF diff --git a/tests/fixtures/encoding/unmapped-glyphs.txt b/tests/fixtures/encoding/unmapped-glyphs.txt new file mode 100644 index 0000000..4e6c581 --- /dev/null +++ b/tests/fixtures/encoding/unmapped-glyphs.txt @@ -0,0 +1 @@ +���� \ No newline at end of file