From 55da9ddf51a828a22afc7baf9000cb927672401e Mon Sep 17 00:00:00 2001 From: jedarden Date: Sun, 5 Jul 2026 13:01:54 -0400 Subject: [PATCH] feat(bf-4cbfb): add community profiles directory and CONTRIBUTING guide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Create profiles/community/ directory structure with CONTRIBUTING.md - CONTRIBUTING.md covers all required contribution areas: * File naming convention and directory structure * Required profile fields and schema requirements * Local testing workflow with validation steps * PR submission process and review guidelines * License requirements for fixture PDFs * Review checklist and quality standards - Update extraction_loader.rs to add Community profile origin - Update profiles_cmd.rs to list community profiles separately - Document community profiles in profile-authoring.md search path - Add test-profile example to verify implementation Acceptance criteria met: ✅ profiles/community/CONTRIBUTING.md exists with comprehensive guide ✅ Code supports loading and listing community profiles ✅ Path documented in profile-authoring.md Closes bf-4cbfb --- crates/pdftract-cli/src/profiles_cmd.rs | 22 ++ .../src/profiles/extraction_loader.rs | 52 ++- docs/research/profile-authoring.md | 22 +- profiles/community/.gitkeep | 2 + profiles/community/CONTRIBUTING.md | 328 ++++++++++++++++++ profiles/community/test-profile/README.md | 21 ++ profiles/community/test-profile/profile.yaml | 18 + 7 files changed, 450 insertions(+), 15 deletions(-) create mode 100644 profiles/community/.gitkeep create mode 100644 profiles/community/CONTRIBUTING.md create mode 100644 profiles/community/test-profile/README.md create mode 100644 profiles/community/test-profile/profile.yaml diff --git a/crates/pdftract-cli/src/profiles_cmd.rs b/crates/pdftract-cli/src/profiles_cmd.rs index 2d95002..e4a6401 100644 --- a/crates/pdftract-cli/src/profiles_cmd.rs +++ b/crates/pdftract-cli/src/profiles_cmd.rs @@ -61,12 +61,14 @@ fn run_list() -> Result<()> { // Group by origin let mut builtin = Vec::new(); + let mut community = Vec::new(); let mut user = Vec::new(); let mut custom = Vec::new(); for source in &profiles { match source.source { extraction_loader::ProfileOrigin::BuiltIn => builtin.push(source), + extraction_loader::ProfileOrigin::Community => community.push(source), extraction_loader::ProfileOrigin::User => user.push(source), extraction_loader::ProfileOrigin::Custom(_) => custom.push(source), extraction_loader::ProfileOrigin::System => { @@ -96,6 +98,26 @@ fn run_list() -> Result<()> { println!(); } + // Print community profiles + if !community.is_empty() { + println!("Community profiles:"); + for source in community { + let profile = &source.profile; + println!( + " {} - Priority: {}{}", + profile.name, + profile.priority, + if source.overrides_builtin { + " (overrides built-in)" + } else { + "" + } + ); + println!(" {}", profile.description); + } + println!(); + } + // Print user profiles if !user.is_empty() { println!("User profiles:"); diff --git a/crates/pdftract-core/src/profiles/extraction_loader.rs b/crates/pdftract-core/src/profiles/extraction_loader.rs index 3c59142..cb519fe 100644 --- a/crates/pdftract-core/src/profiles/extraction_loader.rs +++ b/crates/pdftract-core/src/profiles/extraction_loader.rs @@ -20,6 +20,9 @@ pub struct ProfileSource { /// Whether this overrides a built-in profile pub overrides_builtin: bool, + + /// Whether this overrides a community profile + pub overrides_community: bool, } /// Origin of a profile. @@ -28,6 +31,9 @@ pub enum ProfileOrigin { /// Built-in profile (compiled into binary) BuiltIn, + /// Community profile (profiles/community/) + Community, + /// System-wide profile (/etc/pdftract/profiles/) System, @@ -42,9 +48,10 @@ pub enum ProfileOrigin { /// /// Search order (lowest to highest priority): /// 1. Built-in profiles (compiled in) -/// 2. System directory (/etc/pdftract/profiles/) -/// 3. User directory (XDG config: ~/.config/pdftract/profiles/) -/// 4. Custom directories (--profile-dir, repeatable) +/// 2. Community profiles (profiles/community/) +/// 3. System directory (/etc/pdftract/profiles/) +/// 4. User directory (XDG config: ~/.config/pdftract/profiles/) +/// 5. Custom directories (--profile-dir, repeatable) /// /// Later sources override earlier ones on name collision. pub fn load_extraction_profiles( @@ -55,20 +62,26 @@ pub fn load_extraction_profiles( // 1. Load built-in profiles load_builtin_profiles(&mut profiles_by_name)?; - // 2. Load system profiles + // 2. Load community profiles + let community_dir = PathBuf::from("profiles/community"); + if community_dir.exists() { + load_profiles_from_dir(&community_dir, ProfileOrigin::Community, &mut profiles_by_name)?; + } + + // 3. Load system profiles let system_dir = PathBuf::from("/etc/pdftract/profiles"); if system_dir.exists() { load_profiles_from_dir(&system_dir, ProfileOrigin::System, &mut profiles_by_name)?; } - // 3. Load user profiles (XDG config) + // 4. Load user profiles (XDG config) if let Some(user_dir) = get_xdg_profile_dir() { if user_dir.exists() { load_profiles_from_dir(&user_dir, ProfileOrigin::User, &mut profiles_by_name)?; } } - // 4. Load custom profiles (--profile-dir) + // 5. Load custom profiles (--profile-dir) for custom_dir in custom_dirs { if custom_dir.exists() { let origin = ProfileOrigin::Custom(custom_dir.clone()); @@ -152,6 +165,7 @@ fn load_builtin_profiles( profile, source: ProfileOrigin::BuiltIn, overrides_builtin: false, + overrides_community: false, }, ); } @@ -204,9 +218,15 @@ fn load_profiles_from_dir( let profile_yaml = path.join("profile.yaml"); if profile_yaml.exists() { if let Ok(profile) = load_profile_file(&profile_yaml) { - let overrides_builtin = profiles - .contains_key(&profile.name) - && matches!(origin, ProfileOrigin::User | ProfileOrigin::Custom(_)); + let (overrides_builtin, overrides_community) = if let Some(existing) = profiles.get(&profile.name) { + match &existing.source { + ProfileOrigin::BuiltIn => (true, false), + ProfileOrigin::Community => (false, true), + _ => (false, false), + } + } else { + (false, false) + }; profiles.insert( profile.name.clone(), @@ -214,6 +234,7 @@ fn load_profiles_from_dir( profile, source: origin.clone(), overrides_builtin, + overrides_community, }, ); } @@ -227,9 +248,15 @@ fn load_profiles_from_dir( } if let Ok(profile) = load_profile_file(&path) { - let overrides_builtin = profiles - .contains_key(&profile.name) - && matches!(origin, ProfileOrigin::User | ProfileOrigin::Custom(_)); + let (overrides_builtin, overrides_community) = if let Some(existing) = profiles.get(&profile.name) { + match &existing.source { + ProfileOrigin::BuiltIn => (true, false), + ProfileOrigin::Community => (false, true), + _ => (false, false), + } + } else { + (false, false) + }; profiles.insert( profile.name.clone(), @@ -237,6 +264,7 @@ fn load_profiles_from_dir( profile, source: origin.clone(), overrides_builtin, + overrides_community, }, ); } diff --git a/docs/research/profile-authoring.md b/docs/research/profile-authoring.md index 22ec83a..1eaa243 100644 --- a/docs/research/profile-authoring.md +++ b/docs/research/profile-authoring.md @@ -359,9 +359,25 @@ Error: Invalid combinator structure: Profiles are resolved from multiple sources in order of priority (later sources override earlier ones on name collision): 1. **Built-in profiles** (compiled into binary at `profiles/builtin/`) -2. **System-wide profiles** (`/etc/pdftract/profiles/*.yaml`) -3. **User profiles** (`$XDG_CONFIG_HOME/pdftract/profiles/*.yaml`, defaults to `~/.config/pdftract/profiles/`) -4. **Runtime directories** (`--profile-dir DIR` CLI flag, repeatable) +2. **Community profiles** (`profiles/community/*.yaml` or `profiles/community/*/profile.yaml`) +3. **System-wide profiles** (`/etc/pdftract/profiles/*.yaml`) +4. **User profiles** (`$XDG_CONFIG_HOME/pdftract/profiles/*.yaml`, defaults to `~/.config/pdftract/profiles/`) +5. **Runtime directories** (`--profile-dir DIR` CLI flag, repeatable) + +### 5.1 Community Profiles + +The `profiles/community/` directory contains user-contributed profiles for specialized document types. These are included in the repository and can be used directly: + +```bash +# Use a community profile +pdftract extract --profile profiles/community/my-profile/profile.yaml document.pdf + +# Or copy it to your user config directory +pdftract profiles install profiles/community/my-profile/profile.yaml +pdftract extract --profile my-profile document.pdf +``` + +See [Community Contribution Guide](../../profiles/community/CONTRIBUTING.md) for guidelines on submitting community profiles. ### 5.1 Shadowing Behavior diff --git a/profiles/community/.gitkeep b/profiles/community/.gitkeep new file mode 100644 index 0000000..33a51af --- /dev/null +++ b/profiles/community/.gitkeep @@ -0,0 +1,2 @@ +# This file ensures git tracks this directory +# Community profiles are contributed via pull request and stored here diff --git a/profiles/community/CONTRIBUTING.md b/profiles/community/CONTRIBUTING.md new file mode 100644 index 0000000..059adca --- /dev/null +++ b/profiles/community/CONTRIBUTING.md @@ -0,0 +1,328 @@ +# Contributing Document Profiles + +Thank you for your interest in contributing document profiles to pdftract! Community profiles help make pdftract more useful for everyone by supporting specialized document types and industry-specific formats. + +This guide covers how to create, test, and submit document profiles for inclusion in the pdftract community profile collection. + +## Overview + +Document profiles (pdftract) define how to recognize and extract structured data from specific types of documents (invoices, receipts, contracts, scientific papers, etc.). Each profile specifies: + +- **Match criteria**: How to recognize this document type +- **Extraction strategy**: How to read and parse the document content +- **Field definitions**: What structured data to extract (dates, amounts, line items, etc.) + +Community profiles are stored in `profiles/community/` and can be used by anyone with pdftract. + +## Quick Start + +If you're new to profile authoring, start with an existing profile: + +```bash +# Export a built-in profile to use as a template +pdftract profiles export invoice > my-profile.yaml + +# Edit it to match your document type +vim my-profile.yaml + +# Validate your profile +pdftract profiles validate my-profile.yaml + +# Test it on a sample document +pdftract extract --profile my-profile sample.pdf +``` + +## File Naming Convention + +Profile directories in `profiles/community/` should follow these conventions: + +### Directory Structure + +Each profile should be organized as a directory: + +``` +profiles/community/ +├── .gitkeep # Placeholder file (empty) +├── CONTRIBUTING.md # This file +└── / + ├── profile.yaml # Required: Profile definition + ├── README.md # Required: Profile documentation + └── fixtures/ # Optional: Test documents + ├── sample-01.pdf + └── sample-02.pdf +``` + +### Profile Name Requirements + +- **Use lowercase, hyphen-separated names**: `vendor-invoice`, `w2-form`, `packing-slip` +- **Be specific**: If a profile is for a specific region or industry, include it in the name + - ✅ `eu-invoice-vat`, `medical-billing-statement` + - ❌ `invoice`, `statement` +- **Avoid trademarked terms**: Don't use company names or product names in profile names + - ✅ `saas-subscription-invoice` + - ❌ `salesforce-invoice`, `aws-invoice` + +### File Format + +- **profile.yaml**: YAML format (see [Profile Authoring Guide](../../../docs/research/profile-authoring.md)) +- **README.md**: Markdown format with specific sections (see below) +- **fixtures/**: Sample PDFs for testing (optional but recommended) + +## Required Profile Fields + +Every `profile.yaml` must include these top-level fields: + +```yaml +name: profile-name # Unique identifier (lowercase, hyphens) +description: One-line summary # What this profile extracts +priority: 50 # 1-100, higher = more specific + +match: # Document recognition rules + all: + - ... # All conditions must match + any: + - ... # At least one must match + none: + - ... # None of these may match + +extraction: # Extraction strategy + reading_order: line_dominant + table_detection: strict_borders + # ... (see docs for full schema) + +fields: # Fields to extract + field_name: + type: string|date|decimal|int|bool|array + extraction: + # ... extraction strategy +``` + +### Schema Requirements + +- **name**: Must match directory name +- **description**: Clear, concise description (50-100 characters) +- **priority**: 1-100, where higher values indicate more specific profiles + - 10-30: Generic/broad profiles (e.g., `document`) + - 40-60: Standard document types (e.g., `invoice`, `receipt`) + - 70-100: Highly specific profiles (e.g., `w2-tax-form-2024`) +- **match**: Must define at least one condition +- **fields**: Must define at least one field + +### Validation Rules + +Profiles are automatically validated on submission. Common validation failures: + +- ❌ YAML syntax errors (indentation, unescaped characters) +- ❌ Missing required fields (name, description, priority, match, fields) +- ❌ Forbidden keys (api_key, password, token, secret, credential) +- ❌ Invalid field type combinations (type: decimal with parse: bool) +- ❌ Malformed regex patterns or YAML structure + +See [Profile Authoring Guide](../../../docs/research/profile-authoring.md) for common YAML pitfalls. + +## Local Testing Workflow + +Before submitting a profile, test it thoroughly: + +### 1. Validate the Profile + +```bash +pdftract profiles validate profiles/community//profile.yaml +``` + +Fix any validation errors before proceeding. + +### 2. Test Against Sample Documents + +```bash +# Test extraction on a sample document +pdftract extract --profile profiles/community//profile.yaml sample.pdf + +# Inspect the output +pdftract extract --profile profiles/community//profile.yaml sample.pdf | jq . +``` + +### 3. Test Edge Cases + +Test your profile against: +- ✅ Documents that should match (positive cases) +- ❌ Documents that should NOT match (negative cases) +- 📄 Variations (different layouts, regions, fonts) +- 🚫 Edge cases (missing fields, corrupted pages, blank pages) + +### 4. Performance Check + +Ensure extraction completes in reasonable time: +```bash +time pdftract extract --profile profiles/community//profile.yaml sample.pdf +``` + +Typical targets: +- Simple profiles: < 2 seconds per page +- Complex profiles (tables, multi-page): < 5 seconds per page + +## README.md Template + +Every profile should include a `README.md` with these sections: + +```markdown +# Profile + + + +## Match Criteria Summary + +A document matches this profile when... (describe the key indicators) + +## Extracted Fields + +| Field | Type | Description | Example Value | Source Hint | +|-------|------|-------------|----------------|-------------| +| field1 | string | Description | "example" | regex/near/region | +| field2 | decimal | Description | 123.45 | table: largest_table | + +## Known Limitations + +- Limitation 1 +- Limitation 2 + +## Sample Input + +Example fixtures demonstrating this profile are available in `fixtures/`. + +## Configuration Tips + +(Optional) Tips for customizing or overriding this profile. + +--- + +*This README was auto-generated from `profile.yaml`.* +``` + +See [invoice README](../../../profiles/builtin/invoice/README.md) for a complete example. + +## PR Submission Process + +### 1. Fork and Branch + +```bash +# Fork the repository and create a feature branch +git checkout -b add-profile- +``` + +### 2. Add Your Profile + +```bash +# Create profile directory +mkdir -p profiles/community/ + +# Add profile.yaml and README.md +# ... (edit files) + +# Commit +git add profiles/community/ +git commit -m "Add profile: " +``` + +### 3. Submit Pull Request + +Submit a PR to the main repository with: +- **Title**: `Add profile: ` +- **Description**: Brief summary of what the profile extracts +- **Linked issues**: Reference any related issues or discussions + +### 4. Review Process + +Maintainers will review your profile for: +- ✅ Functional correctness (extraction accuracy) +- ✅ YAML validity and schema compliance +- ✅ Documentation completeness +- ✅ Edge case handling +- ✅ Performance characteristics + +Review typically takes 1-3 business days. + +## License Requirements for Fixtures + +If you include sample PDFs in `fixtures/`, ensure: + +- **Public domain or permissive license**: PDFs must be usable under Apache-2.0 or MIT +- **No personal information**: Remove or redact PII (names, addresses, account numbers) +- **No confidential data**: Don't include real business data, trade secrets, or proprietary information +- **Synthetic data preferred**: Use generated/fabricated sample documents where possible + +### Redaction Examples + +❌ **Don't include unredacted documents**: +``` +Invoice #: INV-2024-001 +Customer: John Doe +Account: 1234-5678-9012 +``` + +✅ **Use synthetic/redacted data**: +``` +Invoice #: INV-DEMO-001 +Customer: [REDACTED] +Account: XXXX-XXXX-XXXX +``` + +## Review Checklist + +Before submitting, confirm: + +- [ ] **Validation passes**: `pdftract profiles validate` returns success +- [ ] **README.md is complete**: Includes all required sections +- [ ] **Field documentation**: Every field has a description and example +- [ ] **Positive cases work**: Profile extracts correctly from intended document types +- [ ] **Negative cases reject**: Profile doesn't match unrelated documents +- [ ] **Edge cases handled**: Gracefully handles missing/empty/malformed fields +- [ ] **Performance acceptable**: Extraction completes in reasonable time +- [ ] **Fixtures licensed**: Sample PDFs use permissive licenses or are synthetic +- [ ] **No secrets**: Profile contains no API keys, passwords, or credentials +- [ ] **Name follows conventions**: Lowercase, hyphen-separated, non-trademarked + +## Profile Quality Standards + +Profiles should meet these quality standards: + +### Accuracy +- Extract correct values in 95%+ of test cases +- Handle common layout variations (centered, left-aligned, multi-column) +- Gracefully degrade when fields are missing + +### Robustness +- Don't crash on malformed input +- Provide sensible defaults when extraction fails +- Log warnings for ambiguous cases + +### Clarity +- Profile name clearly indicates purpose +- Field names are self-documenting +- README explains match criteria and limitations + +### Performance +- No O(n²) or worse algorithms in match logic +- Reasonable memory usage (< 100MB per page) +- Fast enough for batch processing + +## Getting Help + +- **Documentation**: [Profile Authoring Guide](../../../docs/research/profile-authoring.md) +- **Examples**: [Built-in profiles](../../../profiles/builtin/) +- **Issues**: Open a GitHub issue for questions or problems +- **Discussions**: Use GitHub Discussions for design feedback + +## Profile Maintenance + +Profile authors are encouraged to: +- Respond to feedback during review +- Fix bugs reported by users +- Update profiles for new pdftract releases +- Add test fixtures for edge cases + +Maintainers may update profiles for compatibility or correctness. + +--- + +Thank you for contributing to pdftract! 🎉 diff --git a/profiles/community/test-profile/README.md b/profiles/community/test-profile/README.md new file mode 100644 index 0000000..1eb7da4 --- /dev/null +++ b/profiles/community/test-profile/README.md @@ -0,0 +1,21 @@ +# Test Profile + +A test profile for verifying community profile functionality. + +## Match Criteria Summary + +A document matches this profile when it contains the word "test". + +## Extracted Fields + +| Field | Type | Description | Example Value | Source Hint | +|-------|------|-------------|----------------|-------------| +| test_field | string | Extracts text matching "test" pattern | "test value" | regex | + +## Known Limitations + +This is a test profile only. + +## Sample Input + +No fixtures provided for test profile. diff --git a/profiles/community/test-profile/profile.yaml b/profiles/community/test-profile/profile.yaml new file mode 100644 index 0000000..3643dae --- /dev/null +++ b/profiles/community/test-profile/profile.yaml @@ -0,0 +1,18 @@ +name: test-profile +description: Test community profile for verification +priority: 50 + +match: + all: + - text_contains: + patterns: ["test"] + +extraction: + reading_order: line_dominant + table_detection: strict_borders + +fields: + test_field: + type: string + extraction: + regex: "test.*"