pdftract/crates/pdftract-core/benches/cmap_tokenize.rs

//! Benchmark for CJK tokenizer performance.
//!
//! Validates that 100 KB of CJK content stream can be tokenized in < 10 ms.

use criterion::{black_box, BenchmarkId, Criterion, Throughput};
use pdftract_core::cmap::{tokenize_cjk_bytes, CodespaceRange, CodespaceRanges};

fn bench_cjk_tokenization(c: &mut Criterion) {
    let mut group = c.benchmark_group("cmap/tokenize");

    // Create a realistic CJK codespace (1-byte ASCII + 2-byte CJK)
    let mut codespace = CodespaceRanges::new();
    codespace.push(CodespaceRange::new([0x00, 0, 0, 0], [0x7F, 0, 0, 0], 1));
    codespace.push(CodespaceRange::new(
        [0x81, 0x40, 0, 0],
        [0xFE, 0xFE, 0, 0],
        2,
    ));

    // 10 KB of mixed ASCII/CJK content
    let mut small_input = Vec::new();
    for i in 0..5000 {
        if i % 2 == 0 {
            // ASCII
            small_input.push(0x48 + (i % 26) as u8);
        } else {
            // CJK
            small_input.push(0x81 + (i % 30) as u8);
            small_input.push(0x40 + (i % 80) as u8);
        }
    }

    // 100 KB of mixed content
    let mut large_input = Vec::new();
    for i in 0..50000 {
        if i % 2 == 0 {
            // ASCII
            large_input.push(0x48 + (i % 26) as u8);
        } else {
            // CJK
            large_input.push(0x81 + (i % 30) as u8);
            large_input.push(0x40 + (i % 80) as u8);
        }
    }

    group.throughput(Throughput::Bytes(small_input.len() as u64));
    group.bench_with_input(
        BenchmarkId::new("mixed", small_input.len()),
        &small_input,
        |b, input| {
            b.iter(|| {
                let mut diagnostics = Vec::new();
                black_box(tokenize_cjk_bytes(
                    black_box(&codespace),
                    black_box(input),
                    &mut diagnostics,
                ));
            });
        },
    );

    group.throughput(Throughput::Bytes(large_input.len() as u64));
    group.bench_with_input(
        BenchmarkId::new("mixed", large_input.len()),
        &large_input,
        |b, input| {
            b.iter(|| {
                let mut diagnostics = Vec::new();
                black_box(tokenize_cjk_bytes(
                    black_box(&codespace),
                    black_box(input),
                    &mut diagnostics,
                ));
            });
        },
    );

    group.finish();
}

fn bench_empty_codespace(c: &mut Criterion) {
    let mut group = c.benchmark_group("cmap/tokenize/empty_codespace");

    let codespace = CodespaceRanges::new();
    let mut input = vec![0x48; 100_000];

    group.throughput(Throughput::Bytes(input.len() as u64));
    group.bench_function("100KB", |b| {
        b.iter(|| {
            let mut diagnostics = Vec::new();
            black_box(tokenize_cjk_bytes(
                black_box(&codespace),
                black_box(&input),
                &mut diagnostics,
            ));
        });
    });

    group.finish();
}

fn bench_widest_first_matching(c: &mut Criterion) {
    let mut group = c.benchmark_group("cmap/tokenize/widest_first");

    // Create overlapping ranges to test widest-first logic
    let mut codespace = CodespaceRanges::new();
    codespace.push(CodespaceRange::new([0x00, 0, 0, 0], [0xFF, 0, 0, 0], 1));
    codespace.push(CodespaceRange::new(
        [0x80, 0x00, 0, 0],
        [0xFF, 0xFF, 0, 0],
        2,
    ));
    codespace.push(CodespaceRange::new(
        [0x81, 0x40, 0x00, 0],
        [0xFE, 0xFE, 0xFF, 0],
        3,
    ));

    // Input that will match 3-byte sequences
    let mut input = Vec::new();
    for i in 0..20000 {
        input.push(0x81 + (i % 30) as u8);
        input.push(0x40 + (i % 80) as u8);
        input.push(0x00 + (i % 50) as u8);
    }

    group.throughput(Throughput::Bytes(input.len() as u64));
    group.bench_function("3_byte_sequences", |b| {
        b.iter(|| {
            let mut diagnostics = Vec::new();
            black_box(tokenize_cjk_bytes(
                black_box(&codespace),
                black_box(&input),
                &mut diagnostics,
            ));
        });
    });

    group.finish();
}

criterion_group!(
    benches,
    bench_cjk_tokenization,
    bench_empty_codespace,
    bench_widest_first_matching
);
criterion_main!(benches);