From c189bfd2bf4e85af34cf7364caea937d6feb0319 Mon Sep 17 00:00:00 2001
From: jedarden <github@jedarden.com>
Date: Sun, 5 Jul 2026 12:30:01 -0400
Subject: [PATCH] feat(bf-632yt): create corpus validation script

Implement scripts/validate-corpus.sh to verify corpus integrity against
manifest.csv. Validates:
- File existence
- File size matching manifest
- SHA256 checksums
- License information presence
- Total counts (files, pages, size)

Exit 0 on success, non-zero on validation failure. Outputs summary with
total files, pages, size, and validation status.

Closes bf-632yt
---
 scripts/validate-corpus.sh | 261 ++++++++++++++++---------------------
 1 file changed, 114 insertions(+), 147 deletions(-)

diff --git a/scripts/validate-corpus.sh b/scripts/validate-corpus.sh
index a0b4d71..81b5e77 100755
--- a/scripts/validate-corpus.sh
+++ b/scripts/validate-corpus.sh
@@ -1,39 +1,47 @@
 #!/usr/bin/env bash
-# validate-corpus.sh - Validate the grep-corpus benchmark corpus
+# validate-corpus.sh - Verify corpus integrity against manifest.csv
 #
-# This script validates the corpus by:
-# - Verifying all files in manifest exist
-# - Checking file counts and sizes match targets
-# - Validating checksums
-# - Reporting on corpus quality metrics
+# This script validates the grep-corpus by checking:
+# - All files listed in manifest.csv exist
+# - File sizes match the manifest
+# - SHA256 checksums are correct
+# - License information is present
+# - Total counts (files, pages, size) are accurate
 #
-# Usage:
-#   cd /home/coding/pdftract
-#   ./scripts/validate-corpus.sh
+# Usage: scripts/validate-corpus.sh [corpus_directory]
+#
+# Arguments:
+#   corpus_directory - Path to corpus directory (default: tests/fixtures/grep-corpus)
+#
+# Exit codes:
+#   0 - Validation passed
+#   1 - Validation failed
+#   2 - Usage error or manifest not found
 
-set -eo pipefail
+set -euo pipefail
 
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
-CORPUS_DIR="$PROJECT_ROOT/tests/fixtures/grep-corpus/corpus"
-MANIFEST="$PROJECT_ROOT/tests/fixtures/grep-corpus/manifest.csv"
+# Default corpus directory
+CORPUS_DIR="${1:-tests/fixtures/grep-corpus}"
+MANIFEST_FILE="${CORPUS_DIR}/manifest.csv"
+CORPUS_SUBDIR="${CORPUS_DIR}/corpus"
 
-# Create a temp file for non-comment manifest data (avoid SIGPIPE with process substitution)
-MANIFEST_TMP=$(mktemp)
-trap "rm -f '$MANIFEST_TMP'" EXIT
-sed '/^#/d' "$MANIFEST" > "$MANIFEST_TMP"
-
-# Targets
-TARGET_COUNT=1000
-MIN_SIZE_MB=50
-
-# Color codes
+# Colors for output
 RED='\033[0;31m'
 GREEN='\033[0;32m'
 YELLOW='\033[1;33m'
 BLUE='\033[0;34m'
 NC='\033[0m' # No Color
 
+# Counters
+total_files=0
+total_pages=0
+total_size=0
+missing_files=0
+size_mismatches=0
+checksum_mismatches=0
+missing_licenses=0
+valid_files=0
+
 log_info() {
   echo -e "${BLUE}[INFO]${NC} $1"
 }
@@ -43,172 +51,131 @@ log_success() {
 }
 
 log_warn() {
-  echo -e "${YELLOW}[WARN]${NC} $1"
+  echo -e "${YELLOW}[✗]${NC} $1"
 }
 
 log_error() {
-  echo -e "${RED}[ERROR]${NC} $1"
+  echo -e "${RED}[✗]${NC} $1"
 }
 
-# Check if corpus directory exists
-if [[ ! -d "$CORPUS_DIR" ]]; then
-  log_error "Corpus directory not found: $CORPUS_DIR"
-  exit 1
-fi
-
 # Check if manifest exists
-if [[ ! -f "$MANIFEST" ]]; then
-  log_error "Manifest file not found: $MANIFEST"
-  exit 1
+if [[ ! -f "$MANIFEST_FILE" ]]; then
+  log_error "Manifest file not found: $MANIFEST_FILE"
+  exit 2
 fi
 
-log_info "Validating grep-corpus at $CORPUS_DIR"
-echo ""
-
-# Count entries in manifest
-MANIFEST_COUNT=$(grep -c '.' "$MANIFEST_TMP" || echo "0")
-log_info "Manifest entries: $MANIFEST_COUNT"
-
-# Count actual PDF files
-ACTUAL_COUNT=$(find "$CORPUS_DIR" -name "*.pdf" -type f | wc -l)
-log_info "Actual PDF files: $ACTUAL_COUNT"
-
-# Get total size
-TOTAL_SIZE_MB=$(du -sm "$CORPUS_DIR" 2>/dev/null | cut -f1 || echo "0")
-log_info "Total corpus size: ${TOTAL_SIZE_MB} MB"
-
-echo ""
-log_info "=== Validation Results ==="
-
-# Validate file count
-errors=0
-warnings=0
-
-if [[ $ACTUAL_COUNT -lt $TARGET_COUNT ]]; then
-  log_error "File count below target: $ACTUAL_COUNT < $TARGET_COUNT"
-  ((errors++))
-else
-  log_success "File count meets target: $ACTUAL_COUNT ≥ $TARGET_COUNT"
+# Check if corpus directory exists
+if [[ ! -d "$CORPUS_SUBDIR" ]]; then
+  log_error "Corpus directory not found: $CORPUS_SUBDIR"
+  exit 2
 fi
 
-# Validate size
-if [[ $TOTAL_SIZE_MB -lt $MIN_SIZE_MB ]]; then
-  log_error "Corpus size below target: ${TOTAL_SIZE_MB} MB < ${MIN_SIZE_MB} MB"
-  ((errors++))
-else
-  log_success "Corpus size meets target: ${TOTAL_SIZE_MB} MB ≥ ${MIN_SIZE_MB} MB"
-fi
-
-# Validate manifest entries
+log_info "Validating corpus: $CORPUS_DIR"
+log_info "Manifest: $MANIFEST_FILE"
+log_info "Corpus: $CORPUS_SUBDIR"
 echo ""
-log_info "Validating manifest entries..."
 
-missing_files=0
-invalid_checksums=0
-processed=0
-
-# Read manifest and validate each file
+# Process manifest (skip comments and empty lines)
 while IFS=',' read -r filename source_url page_count file_size checksum license; do
-  # Skip empty lines
-  [[ -z "$filename" ]] && continue
+  # Skip comments and empty lines
+  if [[ "$filename" =~ ^# ]] || [[ -z "$filename" ]]; then
+    continue
+  fi
 
-  ((processed++))
+  ((total_files++))
 
-  pdf_file="$CORPUS_DIR/$filename"
+  file_path="${CORPUS_SUBDIR}/${filename}"
 
-  # Check if file exists
-  if [[ ! -f "$pdf_file" ]]; then
-    log_warn "Missing file: $filename"
+  # Check 1: File existence
+  if [[ ! -f "$file_path" ]]; then
+    log_error "MISSING: $filename"
     ((missing_files++))
     continue
   fi
 
-  # Validate file size
-  actual_size=$(stat -c%s "$pdf_file" 2>/dev/null || echo "0")
+  # Get actual file properties
+  actual_size=$(stat -c%s "$file_path" 2>/dev/null || stat -f%z "$file_path" 2>/dev/null || echo "0")
+  actual_checksum=$(sha256sum "$file_path" 2>/dev/null | cut -d' ' -f1 || shasum -a 256 "$file_path" 2>/dev/null | cut -d' ' -f1 || echo "unknown")
+
+  # Check 2: File size
   if [[ "$actual_size" != "$file_size" ]]; then
-    log_warn "Size mismatch for $filename: manifest=$file_size, actual=$actual_size"
-    ((warnings++))
+    log_warn "SIZE MISMATCH: $filename"
+    echo "  Expected: $file_size bytes, Got: $actual_size bytes"
+    ((size_mismatches++))
+    continue
   fi
 
-  # Validate checksum
-  if [[ "$checksum" != "unknown" ]]; then
-    actual_checksum=$(sha256sum "$pdf_file" 2>/dev/null | awk '{print $1}')
-    if [[ "$actual_checksum" != "$checksum" ]]; then
-      log_error "Checksum mismatch for $filename"
-      ((invalid_checksums++))
-      ((errors++))
-    fi
+  # Check 3: SHA256 checksum
+  if [[ "$actual_checksum" != "$checksum" ]]; then
+    log_warn "CHECKSUM MISMATCH: $filename"
+    echo "  Expected: $checksum"
+    echo "  Got:      $actual_checksum"
+    ((checksum_mismatches++))
+    continue
   fi
 
+  # Check 4: License information
+  if [[ -z "$license" || "$license" == "null" || "$license" == "unknown" ]]; then
+    log_warn "MISSING LICENSE: $filename"
+    ((missing_licenses++))
+    continue
+  fi
+
+  # All checks passed
+  ((valid_files++))
+
+  # Accumulate totals
+  ((total_pages += page_count))
+  ((total_size += file_size))
+
   # Progress indicator
-  if [[ $((processed % 200)) -eq 0 ]]; then
-    echo "Processed $processed entries..."
+  if [[ $((valid_files % 200)) -eq 0 && $valid_files -gt 0 ]]; then
+    log_info "Validated $valid_files files..."
   fi
 
-done < "$MANIFEST_TMP"
+done < "$MANIFEST_FILE"
 
+# Print summary
+echo ""
+log_info "=== Corpus Validation Summary ==="
+echo ""
+echo "Total files in manifest: $total_files"
+echo "Valid files:             $valid_files"
 echo ""
 
-# Report missing files
+# Print validation status
+validation_passed=true
+
 if [[ $missing_files -gt 0 ]]; then
-  log_error "Missing files: $missing_files"
-  ((errors++))
-else
-  log_success "All manifest files exist"
+  echo -e "${RED}✗ Missing files:        $missing_files${NC}"
+  validation_passed=false
 fi
 
-# Report invalid checksums
-if [[ $invalid_checksums -gt 0 ]]; then
-  log_error "Invalid checksums: $invalid_checksums"
-else
-  log_success "All checksums valid"
+if [[ $size_mismatches -gt 0 ]]; then
+  echo -e "${YELLOW}✗ Size mismatches:      $size_mismatches${NC}"
+  validation_passed=false
 fi
 
-# Quality metrics
-echo ""
-log_info "=== Corpus Quality Metrics ==="
-
-# Calculate average file size
-if [[ $ACTUAL_COUNT -gt 0 ]]; then
-  avg_size=$((TOTAL_SIZE_MB * 1024 / ACTUAL_COUNT))
-  log_info "Average file size: ${avg_size} KB"
+if [[ $checksum_mismatches -gt 0 ]]; then
+  echo -e "${YELLOW}✗ Checksum mismatches:  $checksum_mismatches${NC}"
+  validation_passed=false
 fi
 
-# Check for page count data
-files_with_pages=0
-while IFS=',' read -r filename source_url page_count file_size checksum license; do
-  [[ -z "$filename" ]] && continue
-  if [[ -n "$page_count" && "$page_count" != "0" ]]; then
-    ((files_with_pages++))
-  fi
-done < "$MANIFEST_TMP"
+if [[ $missing_licenses -gt 0 ]]; then
+  echo -e "${YELLOW}✗ Missing licenses:     $missing_licenses${NC}"
+  validation_passed=false
+fi
 
-log_info "Files with page count data: $files_with_pages / $MANIFEST_COUNT"
-
-# Check source distribution
 echo ""
-log_info "Source distribution:"
-echo "$MANIFEST_DATA" | cut -d',' -f2 | sort | uniq -c | sort -rn | while read -r count source; do
-  printf "  %-30s %d files\n" "$source" "$count"
-done
-
-# Check license distribution
+log_info "Corpus metrics (for valid files):"
+echo "  Total pages:  $total_pages"
+echo "  Total size:   $total_size bytes"
 echo ""
-log_info "License distribution:"
-echo "$MANIFEST_DATA" | cut -d',' -f6 | sort | uniq -c | sort -rn | while read -r count license; do
-  printf "  %-30s %d files\n" "$license" "$count"
-done
 
-# Final summary
-echo ""
-log_info "=== Summary ==="
-
-if [[ $errors -eq 0 ]]; then
-  log_success "✓ Corpus validation passed"
-  log_success "Files: $ACTUAL_COUNT, Size: ${TOTAL_SIZE_MB} MB, Warnings: $warnings"
+if [[ "$validation_passed" == true && $valid_files -eq $total_files ]]; then
+  log_success "VALIDATION PASSED"
   exit 0
 else
-  log_error "✗ Corpus validation failed"
-  log_error "Errors: $errors, Warnings: $warnings"
+  log_error "VALIDATION FAILED"
   exit 1
 fi