feat(bf-632yt): create corpus validation script
Implement scripts/validate-corpus.sh to verify corpus integrity against manifest.csv. Validates: - File existence - File size matching manifest - SHA256 checksums - License information presence - Total counts (files, pages, size) Exit 0 on success, non-zero on validation failure. Outputs summary with total files, pages, size, and validation status. Closes bf-632yt
This commit is contained in:
jedarden
parent
9a8252e92d
commit
c189bfd2bf
1 changed files with 114 additions and 147 deletions
|
|
@ -1,39 +1,47 @@
|
|||
#!/usr/bin/env bash
|
||||
# validate-corpus.sh - Validate the grep-corpus benchmark corpus
|
||||
# validate-corpus.sh - Verify corpus integrity against manifest.csv
|
||||
#
|
||||
# This script validates the corpus by:
|
||||
# - Verifying all files in manifest exist
|
||||
# - Checking file counts and sizes match targets
|
||||
# - Validating checksums
|
||||
# - Reporting on corpus quality metrics
|
||||
# This script validates the grep-corpus by checking:
|
||||
# - All files listed in manifest.csv exist
|
||||
# - File sizes match the manifest
|
||||
# - SHA256 checksums are correct
|
||||
# - License information is present
|
||||
# - Total counts (files, pages, size) are accurate
|
||||
#
|
||||
# Usage:
|
||||
# cd /home/coding/pdftract
|
||||
# ./scripts/validate-corpus.sh
|
||||
# Usage: scripts/validate-corpus.sh [corpus_directory]
|
||||
#
|
||||
# Arguments:
|
||||
# corpus_directory - Path to corpus directory (default: tests/fixtures/grep-corpus)
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 - Validation passed
|
||||
# 1 - Validation failed
|
||||
# 2 - Usage error or manifest not found
|
||||
|
||||
set -eo pipefail
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
CORPUS_DIR="$PROJECT_ROOT/tests/fixtures/grep-corpus/corpus"
|
||||
MANIFEST="$PROJECT_ROOT/tests/fixtures/grep-corpus/manifest.csv"
|
||||
# Default corpus directory
|
||||
CORPUS_DIR="${1:-tests/fixtures/grep-corpus}"
|
||||
MANIFEST_FILE="${CORPUS_DIR}/manifest.csv"
|
||||
CORPUS_SUBDIR="${CORPUS_DIR}/corpus"
|
||||
|
||||
# Create a temp file for non-comment manifest data (avoid SIGPIPE with process substitution)
|
||||
MANIFEST_TMP=$(mktemp)
|
||||
trap "rm -f '$MANIFEST_TMP'" EXIT
|
||||
sed '/^#/d' "$MANIFEST" > "$MANIFEST_TMP"
|
||||
|
||||
# Targets
|
||||
TARGET_COUNT=1000
|
||||
MIN_SIZE_MB=50
|
||||
|
||||
# Color codes
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Counters
|
||||
total_files=0
|
||||
total_pages=0
|
||||
total_size=0
|
||||
missing_files=0
|
||||
size_mismatches=0
|
||||
checksum_mismatches=0
|
||||
missing_licenses=0
|
||||
valid_files=0
|
||||
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
|
@ -43,172 +51,131 @@ log_success() {
|
|||
}
|
||||
|
||||
log_warn() {
|
||||
echo -e "${YELLOW}[WARN]${NC} $1"
|
||||
echo -e "${YELLOW}[✗]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
echo -e "${RED}[✗]${NC} $1"
|
||||
}
|
||||
|
||||
# Check if corpus directory exists
|
||||
if [[ ! -d "$CORPUS_DIR" ]]; then
|
||||
log_error "Corpus directory not found: $CORPUS_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if manifest exists
|
||||
if [[ ! -f "$MANIFEST" ]]; then
|
||||
log_error "Manifest file not found: $MANIFEST"
|
||||
exit 1
|
||||
if [[ ! -f "$MANIFEST_FILE" ]]; then
|
||||
log_error "Manifest file not found: $MANIFEST_FILE"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
log_info "Validating grep-corpus at $CORPUS_DIR"
|
||||
echo ""
|
||||
|
||||
# Count entries in manifest
|
||||
MANIFEST_COUNT=$(grep -c '.' "$MANIFEST_TMP" || echo "0")
|
||||
log_info "Manifest entries: $MANIFEST_COUNT"
|
||||
|
||||
# Count actual PDF files
|
||||
ACTUAL_COUNT=$(find "$CORPUS_DIR" -name "*.pdf" -type f | wc -l)
|
||||
log_info "Actual PDF files: $ACTUAL_COUNT"
|
||||
|
||||
# Get total size
|
||||
TOTAL_SIZE_MB=$(du -sm "$CORPUS_DIR" 2>/dev/null | cut -f1 || echo "0")
|
||||
log_info "Total corpus size: ${TOTAL_SIZE_MB} MB"
|
||||
|
||||
echo ""
|
||||
log_info "=== Validation Results ==="
|
||||
|
||||
# Validate file count
|
||||
errors=0
|
||||
warnings=0
|
||||
|
||||
if [[ $ACTUAL_COUNT -lt $TARGET_COUNT ]]; then
|
||||
log_error "File count below target: $ACTUAL_COUNT < $TARGET_COUNT"
|
||||
((errors++))
|
||||
else
|
||||
log_success "File count meets target: $ACTUAL_COUNT ≥ $TARGET_COUNT"
|
||||
# Check if corpus directory exists
|
||||
if [[ ! -d "$CORPUS_SUBDIR" ]]; then
|
||||
log_error "Corpus directory not found: $CORPUS_SUBDIR"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Validate size
|
||||
if [[ $TOTAL_SIZE_MB -lt $MIN_SIZE_MB ]]; then
|
||||
log_error "Corpus size below target: ${TOTAL_SIZE_MB} MB < ${MIN_SIZE_MB} MB"
|
||||
((errors++))
|
||||
else
|
||||
log_success "Corpus size meets target: ${TOTAL_SIZE_MB} MB ≥ ${MIN_SIZE_MB} MB"
|
||||
fi
|
||||
|
||||
# Validate manifest entries
|
||||
log_info "Validating corpus: $CORPUS_DIR"
|
||||
log_info "Manifest: $MANIFEST_FILE"
|
||||
log_info "Corpus: $CORPUS_SUBDIR"
|
||||
echo ""
|
||||
log_info "Validating manifest entries..."
|
||||
|
||||
missing_files=0
|
||||
invalid_checksums=0
|
||||
processed=0
|
||||
|
||||
# Read manifest and validate each file
|
||||
# Process manifest (skip comments and empty lines)
|
||||
while IFS=',' read -r filename source_url page_count file_size checksum license; do
|
||||
# Skip empty lines
|
||||
[[ -z "$filename" ]] && continue
|
||||
# Skip comments and empty lines
|
||||
if [[ "$filename" =~ ^# ]] || [[ -z "$filename" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
((processed++))
|
||||
((total_files++))
|
||||
|
||||
pdf_file="$CORPUS_DIR/$filename"
|
||||
file_path="${CORPUS_SUBDIR}/${filename}"
|
||||
|
||||
# Check if file exists
|
||||
if [[ ! -f "$pdf_file" ]]; then
|
||||
log_warn "Missing file: $filename"
|
||||
# Check 1: File existence
|
||||
if [[ ! -f "$file_path" ]]; then
|
||||
log_error "MISSING: $filename"
|
||||
((missing_files++))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Validate file size
|
||||
actual_size=$(stat -c%s "$pdf_file" 2>/dev/null || echo "0")
|
||||
# Get actual file properties
|
||||
actual_size=$(stat -c%s "$file_path" 2>/dev/null || stat -f%z "$file_path" 2>/dev/null || echo "0")
|
||||
actual_checksum=$(sha256sum "$file_path" 2>/dev/null | cut -d' ' -f1 || shasum -a 256 "$file_path" 2>/dev/null | cut -d' ' -f1 || echo "unknown")
|
||||
|
||||
# Check 2: File size
|
||||
if [[ "$actual_size" != "$file_size" ]]; then
|
||||
log_warn "Size mismatch for $filename: manifest=$file_size, actual=$actual_size"
|
||||
((warnings++))
|
||||
log_warn "SIZE MISMATCH: $filename"
|
||||
echo " Expected: $file_size bytes, Got: $actual_size bytes"
|
||||
((size_mismatches++))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Validate checksum
|
||||
if [[ "$checksum" != "unknown" ]]; then
|
||||
actual_checksum=$(sha256sum "$pdf_file" 2>/dev/null | awk '{print $1}')
|
||||
if [[ "$actual_checksum" != "$checksum" ]]; then
|
||||
log_error "Checksum mismatch for $filename"
|
||||
((invalid_checksums++))
|
||||
((errors++))
|
||||
fi
|
||||
# Check 3: SHA256 checksum
|
||||
if [[ "$actual_checksum" != "$checksum" ]]; then
|
||||
log_warn "CHECKSUM MISMATCH: $filename"
|
||||
echo " Expected: $checksum"
|
||||
echo " Got: $actual_checksum"
|
||||
((checksum_mismatches++))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Check 4: License information
|
||||
if [[ -z "$license" || "$license" == "null" || "$license" == "unknown" ]]; then
|
||||
log_warn "MISSING LICENSE: $filename"
|
||||
((missing_licenses++))
|
||||
continue
|
||||
fi
|
||||
|
||||
# All checks passed
|
||||
((valid_files++))
|
||||
|
||||
# Accumulate totals
|
||||
((total_pages += page_count))
|
||||
((total_size += file_size))
|
||||
|
||||
# Progress indicator
|
||||
if [[ $((processed % 200)) -eq 0 ]]; then
|
||||
echo "Processed $processed entries..."
|
||||
if [[ $((valid_files % 200)) -eq 0 && $valid_files -gt 0 ]]; then
|
||||
log_info "Validated $valid_files files..."
|
||||
fi
|
||||
|
||||
done < "$MANIFEST_TMP"
|
||||
done < "$MANIFEST_FILE"
|
||||
|
||||
# Print summary
|
||||
echo ""
|
||||
log_info "=== Corpus Validation Summary ==="
|
||||
echo ""
|
||||
echo "Total files in manifest: $total_files"
|
||||
echo "Valid files: $valid_files"
|
||||
echo ""
|
||||
|
||||
# Report missing files
|
||||
# Print validation status
|
||||
validation_passed=true
|
||||
|
||||
if [[ $missing_files -gt 0 ]]; then
|
||||
log_error "Missing files: $missing_files"
|
||||
((errors++))
|
||||
else
|
||||
log_success "All manifest files exist"
|
||||
echo -e "${RED}✗ Missing files: $missing_files${NC}"
|
||||
validation_passed=false
|
||||
fi
|
||||
|
||||
# Report invalid checksums
|
||||
if [[ $invalid_checksums -gt 0 ]]; then
|
||||
log_error "Invalid checksums: $invalid_checksums"
|
||||
else
|
||||
log_success "All checksums valid"
|
||||
if [[ $size_mismatches -gt 0 ]]; then
|
||||
echo -e "${YELLOW}✗ Size mismatches: $size_mismatches${NC}"
|
||||
validation_passed=false
|
||||
fi
|
||||
|
||||
# Quality metrics
|
||||
echo ""
|
||||
log_info "=== Corpus Quality Metrics ==="
|
||||
|
||||
# Calculate average file size
|
||||
if [[ $ACTUAL_COUNT -gt 0 ]]; then
|
||||
avg_size=$((TOTAL_SIZE_MB * 1024 / ACTUAL_COUNT))
|
||||
log_info "Average file size: ${avg_size} KB"
|
||||
if [[ $checksum_mismatches -gt 0 ]]; then
|
||||
echo -e "${YELLOW}✗ Checksum mismatches: $checksum_mismatches${NC}"
|
||||
validation_passed=false
|
||||
fi
|
||||
|
||||
# Check for page count data
|
||||
files_with_pages=0
|
||||
while IFS=',' read -r filename source_url page_count file_size checksum license; do
|
||||
[[ -z "$filename" ]] && continue
|
||||
if [[ -n "$page_count" && "$page_count" != "0" ]]; then
|
||||
((files_with_pages++))
|
||||
fi
|
||||
done < "$MANIFEST_TMP"
|
||||
if [[ $missing_licenses -gt 0 ]]; then
|
||||
echo -e "${YELLOW}✗ Missing licenses: $missing_licenses${NC}"
|
||||
validation_passed=false
|
||||
fi
|
||||
|
||||
log_info "Files with page count data: $files_with_pages / $MANIFEST_COUNT"
|
||||
|
||||
# Check source distribution
|
||||
echo ""
|
||||
log_info "Source distribution:"
|
||||
echo "$MANIFEST_DATA" | cut -d',' -f2 | sort | uniq -c | sort -rn | while read -r count source; do
|
||||
printf " %-30s %d files\n" "$source" "$count"
|
||||
done
|
||||
|
||||
# Check license distribution
|
||||
log_info "Corpus metrics (for valid files):"
|
||||
echo " Total pages: $total_pages"
|
||||
echo " Total size: $total_size bytes"
|
||||
echo ""
|
||||
log_info "License distribution:"
|
||||
echo "$MANIFEST_DATA" | cut -d',' -f6 | sort | uniq -c | sort -rn | while read -r count license; do
|
||||
printf " %-30s %d files\n" "$license" "$count"
|
||||
done
|
||||
|
||||
# Final summary
|
||||
echo ""
|
||||
log_info "=== Summary ==="
|
||||
|
||||
if [[ $errors -eq 0 ]]; then
|
||||
log_success "✓ Corpus validation passed"
|
||||
log_success "Files: $ACTUAL_COUNT, Size: ${TOTAL_SIZE_MB} MB, Warnings: $warnings"
|
||||
if [[ "$validation_passed" == true && $valid_files -eq $total_files ]]; then
|
||||
log_success "VALIDATION PASSED"
|
||||
exit 0
|
||||
else
|
||||
log_error "✗ Corpus validation failed"
|
||||
log_error "Errors: $errors, Warnings: $warnings"
|
||||
log_error "VALIDATION FAILED"
|
||||
exit 1
|
||||
fi
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue