From bbbb7640d49e2a2756b55c46bb41bf200c2a79f1 Mon Sep 17 00:00:00 2001 From: jedarden Date: Sun, 5 Jul 2026 20:14:39 -0400 Subject: [PATCH] chore(bf-xqib3): remove obsolete generators and compiled artifacts from tests/fixtures/ Remove 5 DELETE-category generator files identified in bf-1iefu: - tests/fixtures/scanned/generate_scanned_fixtures.rs (Rust stub, use Python version) - tests/fixtures/security/generate_sensitive_fixture.py (duplicate, Rust version preferred) - tests/fixtures/encoding/generate_unmapped_glyphs.rs (superseded by Python version) - tests/fixtures/malformed/gen-bomb-10k-2g.sh (superseded by Python version) - tests/fixtures/forms/generate_form_fixtures.d (orphaned Makefile dependency) Verification: No compiled object files (.o, .so, .a, .dylib, .dll) or executable binaries found. The .bin files in fixtures root are test data files, not compiled artifacts. --- .../encoding/generate_unmapped_glyphs.rs | 87 ------------ tests/fixtures/forms/generate_form_fixtures.d | 5 - tests/fixtures/malformed/gen-bomb-10k-2g.sh | 109 --------------- .../scanned/generate_scanned_fixtures.rs | 118 ---------------- .../security/generate_sensitive_fixture.py | 127 ------------------ 5 files changed, 446 deletions(-) delete mode 100644 tests/fixtures/encoding/generate_unmapped_glyphs.rs delete mode 100644 tests/fixtures/forms/generate_form_fixtures.d delete mode 100644 tests/fixtures/malformed/gen-bomb-10k-2g.sh delete mode 100644 tests/fixtures/scanned/generate_scanned_fixtures.rs delete mode 100644 tests/fixtures/security/generate_sensitive_fixture.py diff --git a/tests/fixtures/encoding/generate_unmapped_glyphs.rs b/tests/fixtures/encoding/generate_unmapped_glyphs.rs deleted file mode 100644 index bbdc570..0000000 --- a/tests/fixtures/encoding/generate_unmapped_glyphs.rs +++ /dev/null @@ -1,87 +0,0 @@ -//! Generate unmapped-glyphs.pdf test fixture -//! -//! Creates a PDF with custom glyph names that are not in the Adobe Glyph List -//! and have no ToUnicode mapping, triggering GLYPH_UNMAPPED diagnostics. - -use std::fs::File; -use std::io::Write; - -fn main() -> Result<(), Box> { - println!("Generating unmapped-glyphs.pdf..."); - - let objects = vec![ - // Catalog - b"1 0 obj\n<<\n/Type /Catalog\n/Pages 2 0 R\n>>\nendobj".to_vec(), - - // Pages - b"2 0 obj\n<<\n/Type /Pages\n/Kids [3 0 R]\n/Count 1\n>>\nendobj".to_vec(), - - // Page - b"3 0 obj\n<<\n/Type /Page\n/Parent 2 0 R\n/MediaBox [0 0 612 792]\n/Resources <<\n/Font <<\n/F1 5 0 R\n>>\n>>\n/Contents 4 0 R\n>>\nendobj".to_vec(), - - // Content stream: <00010203> Tj (byte codes 0-3 for 4 unmapped glyphs) - b"4 0 obj\n<<\n/Length 44\n>>\nstream\nBT\n/F1 12 Tf\n50 700 Td\n<00010203> Tj\nET\nendstream\nendobj".to_vec(), - - // Font with custom encoding - // Uses 4 distinct custom glyph names not in AGL: - // /CustomAlpha, /CustomBeta, /CustomGamma, /CustomDelta - b"5 0 obj\n<<\n/Type /Font\n/Subtype /Type1\n/BaseFont /UnmappedGlyphs\n/Encoding <<\n/Type /Encoding\n/Differences [0 /CustomAlpha /CustomBeta /CustomGamma /CustomDelta]\n>>\n>>\nendobj".to_vec(), - ]; - - let pdf = build_pdf(objects); - - let mut file = File::create("tests/fixtures/encoding/unmapped-glyphs.pdf")?; - file.write_all(&pdf)?; - - // Create expected output file (4 U+FFFD replacement characters) - let mut txt_file = File::create("tests/fixtures/encoding/unmapped-glyphs.txt")?; - txt_file.write_all("\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}".as_bytes())?; - - println!("✓ unmapped-glyphs.pdf created successfully!"); - println!(" -> Contains 4 unmapped glyphs: /CustomAlpha, /CustomBeta, /CustomGamma, /CustomDelta"); - println!(" -> Expected output: "); - - Ok(()) -} - -/// Build a complete PDF with proper xref table and trailer -fn build_pdf(objects: Vec>) -> Vec { - let mut pdf = b"%PDF-1.4\n".to_vec(); - - // Track object offsets - let mut offsets = Vec::new(); - for obj in &objects { - offsets.push(pdf.len()); - pdf.extend(obj); - pdf.extend(b"\n"); - } - - let xref_start = pdf.len(); - - // Build xref table - pdf.extend(b"xref\n"); - pdf.extend(format!("0 {}\n", objects.len() + 1).as_bytes()); - pdf.extend(b"0000000000 65535 f \n"); - - for offset in offsets { - pdf.extend(format!("{:010} 00000 n \n", offset).as_bytes()); - } - - // Build trailer - pdf.extend(b"trailer\n"); - pdf.extend(b"<<\n"); - pdf.extend(b"/Size "); - pdf.extend(format!("{}", objects.len() + 1).as_bytes()); - pdf.extend(b"\n"); - pdf.extend(b"/Root 1 0 R\n"); - pdf.extend(b">>\n"); - - // Start of xref - pdf.extend(b"startxref\n"); - pdf.extend(format!("{}\n", xref_start).as_bytes()); - - // EOF marker - pdf.extend(b"%%EOF\n"); - - pdf -} diff --git a/tests/fixtures/forms/generate_form_fixtures.d b/tests/fixtures/forms/generate_form_fixtures.d deleted file mode 100644 index bd096be..0000000 --- a/tests/fixtures/forms/generate_form_fixtures.d +++ /dev/null @@ -1,5 +0,0 @@ -generate_form_fixtures.d: generate_form_fixtures.rs - -generate_form_fixtures: generate_form_fixtures.rs - -generate_form_fixtures.rs: diff --git a/tests/fixtures/malformed/gen-bomb-10k-2g.sh b/tests/fixtures/malformed/gen-bomb-10k-2g.sh deleted file mode 100644 index 94b1508..0000000 --- a/tests/fixtures/malformed/gen-bomb-10k-2g.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env bash -# Generate tests/fixtures/malformed/bomb-10k-2g.pdf -# -# This PDF contains a FlateDecode stream that is ~10 KB compressed -# but expands to ~2 GB when decompressed (decompression bomb). -# -# Generation method: -# 1. Create a minimal valid PDF structure -# 2. Include a FlateDecode-compressed stream with highly repetitive data -# 3. The repetitive data (e.g., 0x00 repeated) compresses to ~10KB -# 4. When decompressed, it expands to ~2GB of zeros -# -# This is a TH-01 test fixture for decompression bomb protection. - -set -euo pipefail - -# Output path -OUTPUT_DIR="$(dirname "$0")" -OUTPUT="$OUTPUT_DIR/bomb-10k-2g.pdf" - -# Create a temporary directory for the compressed stream -TEMP_DIR=$(mktemp -d) -trap "rm -rf $TEMP_DIR" EXIT - -# Generate 2GB of zeros and compress them -# This creates the "bomb": small compressed size, huge decompressed size -# We use /dev/zero which compresses extremely well -echo "Generating 2GB bomb stream (this may take a moment)..." -dd if=/dev/zero bs=1M count=2048 2>/dev/null | \ - zlib-flate -compress > "$TEMP_DIR/bomb-stream.bin" - -# Check compressed size is reasonable (~10KB target) -COMPRESSED_SIZE=$(stat -f%z "$TEMP_DIR/bomb-stream.bin" 2>/dev/null || stat -c%s "$TEMP_DIR/bomb-stream.bin" 2>/dev/null) -echo "Compressed stream size: $COMPRESSED_SIZE bytes" - -# Create the PDF structure -# We use a minimal PDF with a single page containing the bomb stream -cat > "$OUTPUT" <<'EOF' -%PDF-1.4 -1 0 obj -<< -/Type /Catalog -/Pages 2 0 R ->> -endobj -2 0 obj -<< -/Type /Pages -/Kids [3 0 R] -/Count 1 ->> -endobj -3 0 obj -<< -/Type /Page -/Parent 2 0 R -/MediaBox [0 0 612 792] -/Contents 4 0 R ->> -endobj -4 0 obj -<< -/Length STREAM_LENGTH -/Filter /FlateDecode ->> -stream -EOF - -# Append the compressed bomb stream -cat "$TEMP_DIR/bomb-stream.bin" >> "$OUTPUT" - -# Close the stream and add the PDF trailer -cat >> "$OUTPUT" <<'EOF' -endstream -endobj -xref -0 5 -0000000000 65535 f -0000000009 00000 n -0000000058 00000 n -0000000115 00000 n -0000000214 00000 n -trailer -<< -/Size 5 -/Root 1 0 R ->> -startxref -STREAM_OFFSET -%%EOF -EOF - -# Replace placeholders with actual values -STREAM_LENGTH=$COMPRESSED_SIZE -# Calculate the offset of the startxref value -# This is the byte offset of the "stream" keyword + length of "stream\r\n" -# We need to be precise here for a valid PDF -STREAM_OFFSET=$(grep -abo "stream$" "$OUTPUT" | head -1 | cut -d: -f1) -STREAM_OFFSET=$((STREAM_OFFSET + 7)) - -# Update the Length and startxref values -sed -i.bak -e "s/STREAM_LENGTH/$STREAM_LENGTH/g" "$OUTPUT" -sed -i.bak -e "s/STREAM_OFFSET/$STREAM_OFFSET/g" "$OUTPUT" -rm -f "$OUTPUT.bak" - -echo "Generated $OUTPUT" -echo "Compressed size: $COMPRESSED_SIZE bytes" -echo "Decompressed size: 2147483648 bytes (2 GB)" -echo "Compression ratio: ~214748:1" diff --git a/tests/fixtures/scanned/generate_scanned_fixtures.rs b/tests/fixtures/scanned/generate_scanned_fixtures.rs deleted file mode 100644 index 3f84d1c..0000000 --- a/tests/fixtures/scanned/generate_scanned_fixtures.rs +++ /dev/null @@ -1,118 +0,0 @@ -//! Generate scanned fixture PDFs from ground truth text files. -//! -//! This is a Rust-native alternative to the Python generator. -//! Run with: cargo run --bin generate_scanned_fixtures - -use std::fs::{self, File}; -use std::io::{BufWriter, Write}; -use std::path::Path; - -fn main() -> Result<(), Box> { - println!("Generating scanned fixture metadata..."); - - // Ensure directories exist - create_directories()?; - - // Generate fixture metadata - generate_fixture_metadata()?; - - println!("\nScanned fixtures corpus structure created."); - println!("\nNOTE: Actual PDF generation requires external tools."); - println!("Options:"); - println!(" 1. Use Python script: generate_scanned_fixtures.py"); - println!(" 2. Manual generation (see GEN_MANIFEST.md)"); - println!(" 3. Use printpdf or similar crate for native Rust generation"); - - Ok(()) -} - -fn create_directories() -> Result<(), Box> { - let dirs = [ - "tests/fixtures/scanned/receipt", - "tests/fixtures/scanned/documents", - "tests/fixtures/scanned/multi-page", - ]; - - for dir in &dirs { - fs::create_dir_all(dir)?; - println!("Created directory: {}", dir); - } - - Ok(()) -} - -fn generate_fixture_metadata() -> Result<(), Box> { - // Create a simple fixture list for reference - let fixtures = vec![ - FixtureSpec { - name: "receipt-300dpi", - dir: "receipt", - font: "Helvetica", - font_size: 10, - pages: 1, - wer_target: 3.0, - }, - FixtureSpec { - name: "invoice-300dpi", - dir: "documents", - font: "Helvetica", - font_size: 11, - pages: 1, - wer_target: 3.0, - }, - FixtureSpec { - name: "form-300dpi", - dir: "documents", - font: "Helvetica", - font_size: 11, - pages: 1, - wer_target: 3.0, - }, - FixtureSpec { - name: "doc-10page-300dpi", - dir: "multi-page", - font: "Times-Roman", - font_size: 12, - pages: 10, - wer_target: 3.0, - }, - ]; - - let manifest_path = "tests/fixtures/scanned/.fixtures.json"; - let file = File::create(manifest_path)?; - let mut writer = BufWriter::new(file); - - writeln!(writer, "{{")?; - writeln!(writer, " \"fixtures\": [")?; - - for (i, fixture) in fixtures.iter().enumerate() { - writeln!( - writer, - " {}{{", - if i == 0 { "" } else { ",\n" } - )?; - writeln!(writer, r#" "name": "{}","#, fixture.name)?; - writeln!(writer, r#" "dir": "{}","#, fixture.dir)?; - writeln!(writer, r#" "font": "{}","#, fixture.font)?; - writeln!(writer, r#" "font_size": {},"#, fixture.font_size)?; - writeln!(writer, r#" "pages": {},"#, fixture.pages)?; - writeln!(writer, r#" "wer_target": {}"#, fixture.wer_target)?; - write!(writer, " }}")?; - } - - writeln!(writer, "\n ]")?; - writeln!(writer, "}}")?; - - println!("Created fixture manifest: {}", manifest_path); - - Ok(()) -} - -struct FixtureSpec<'a> { - name: &'a str, - dir: &'a str, - font: &'a str, - font_size: u32, - pages: u32, - wer_target: f64, -} diff --git a/tests/fixtures/security/generate_sensitive_fixture.py b/tests/fixtures/security/generate_sensitive_fixture.py deleted file mode 100644 index 73ba722..0000000 --- a/tests/fixtures/security/generate_sensitive_fixture.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python3 -""" -Generate sensitive.pdf for TH-08 log audit test. - -This script creates a password-protected PDF with unique, distinctive markers: -- Body text contains "UNIQUE-MARKER-IN-BODY-TEXT-7f9a" -- Password value is "UNIQUE-PASSWORD-FOR-TH08-7f9a" - -These markers are specifically designed to be unlikely to appear -in normal log output, making substring-based leak detection reliable. -""" - -import pikepdf -import io - -# Constants for unique markers -BODY_TEXT = "UNIQUE-MARKER-IN-BODY-TEXT-7f9a" -PASSWORD = "UNIQUE-PASSWORD-FOR-TH08-7f9a" - -# Minimal PDF content with the unique marker -MINIMAL_PDF = f"""%PDF-1.4 -1 0 obj -<< -/Type /Catalog -/Pages 2 0 R ->> -endobj -2 0 obj -<< -/Type /Pages -/Count 1 -/Kids [3 0 R] ->> -endobj -3 0 obj -<< -/Type /Page -/Parent 2 0 R -/MediaBox [0 0 612 792] -/Resources << -/Font << -/F1 << -/Type /Font -/Subtype /Type1 -/BaseFont /Helvetica ->> ->> ->> -/Contents 4 0 R ->> -endobj -4 0 obj -<< -/Length {len(BODY_TEXT) + 30} ->> -stream -BT -/F1 12 Tf -100 700 Td -({BODY_TEXT}) Tj -ET -endstream -endobj -xref -0 5 -0000000000 65535 f -0000000009 00000 n -0000000058 00000 n -0000000115 00000 n -0000000350 00000 n -trailer -<< -/Size 5 -/Root 1 0 R ->> -startxref -450 -%%EOF -""" - -def create_sensitive_pdf(): - """Create a password-protected PDF with unique markers.""" - # Load the minimal PDF from bytes - base_pdf = pikepdf.open(io.BytesIO(MINIMAL_PDF.encode())) - - # Save with password protection - output_path = "tests/fixtures/security/sensitive.pdf" - base_pdf.save( - output_path, - encryption=pikepdf.Encryption( - owner="", - user=PASSWORD, - R=2, # RC4-40 (widest compatibility) - aes=False, # RC4 encryption for R=2 - allow=pikepdf.Permissions( - accessibility=True, - extract=True, - modify_annotation=True, - modify_assembly=False, - modify_form=True, - modify_other=True, - print_lowres=True, - print_highres=True - ), - metadata=False # Can't encrypt metadata with R < 4 - ) - ) - - print(f"Created {output_path}") - print(f" Password: {PASSWORD}") - print(f" Body text marker: {BODY_TEXT}") - -if __name__ == "__main__": - import os - - # Create security fixtures directory if it doesn't exist - os.makedirs("tests/fixtures/security", exist_ok=True) - - try: - create_sensitive_pdf() - print("\nSensitive fixture created successfully for TH-08 log audit test!") - except Exception as e: - print(f"Error: {e}") - import traceback - traceback.print_exc() - print("\nNote: This script requires pikepdf.") - print("Install with: pip install pikepdf")