diff --git a/.ci/argo-workflows/pdftract-build-binaries.yaml b/.ci/argo-workflows/pdftract-build-binaries.yaml new file mode 100644 index 0000000..5818c6a --- /dev/null +++ b/.ci/argo-workflows/pdftract-build-binaries.yaml @@ -0,0 +1,697 @@ +# pdftract-build-binaries WorkflowTemplate +# +# Builds 10 binary archives: 5 target triples × 2 feature variants (default, full) +# Triggered by milestone tag (vX.Y.Z). Archives are uploaded as Argo artifacts +# for the downstream pdftract-github-release template to attach to the GitHub Release. +# +# Target triples: +# - x86_64-unknown-linux-musl +# - aarch64-unknown-linux-musl +# - x86_64-apple-darwin +# - aarch64-apple-darwin +# - x86_64-pc-windows-gnu +# +# Feature variants: +# - default (archive name: pdftract-vX.Y.Z-.tar.gz) +# - full (archive name: pdftract-full-vX.Y.Z-.tar.gz) +# +# Bead: pdftract-220e +# Plan section: Release Engineering / Argo WorkflowTemplates, line 3389 +# +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: pdftract-build-binaries + namespace: argo-workflows + labels: + app.kubernetes.io/name: pdftract-build-binaries + app.kubernetes.io/component: ci + app.kubernetes.io/part-of: pdftract +spec: + entrypoint: build-matrix + serviceAccountName: argo-workflow + + podGC: + strategy: OnPodCompletion + + ttlStrategy: + secondsAfterSuccess: 1800 + secondsAfterFailure: 7200 + + arguments: + parameters: + - name: repo + value: jedarden/pdftract + - name: branch + value: main + - name: tag + value: "" + description: "Git tag (e.g., v0.1.0)" + - name: version + value: "" + description: "Version extracted from tag (e.g., 0.1.0)" + - name: commit-sha + value: "" + description: "Full commit SHA of the tag" + + volumeClaimTemplates: + - metadata: + name: cargo-cache + spec: + accessModes: [ReadWriteOnce] + storageClassName: sata-large + resources: + requests: + storage: 50Gi + - metadata: + name: workspace + spec: + accessModes: [ReadWriteOnce] + storageClassName: sata-large + resources: + requests: + storage: 10Gi + - metadata: + name: build-artifacts + spec: + accessModes: [ReadWriteOnce] + storageClassName: sata-large + resources: + requests: + storage: 5Gi + + volumes: + - name: docker-config + secret: + secretName: docker-hub-registry + items: + - key: .dockerconfigjson + path: config.json + # === osxcross SDK Secret Volume === + # The osxcross-sdk Secret contains the macOS SDK tarball. + # This Secret is pre-created in the argo-workflows namespace. + - name: osxcross-sdk + secret: + secretName: osxcross-sdk + optional: true + + podMetadata: + labels: + app.kubernetes.io/name: pdftract-build-binaries + tag: "{{workflow.parameters.tag}}" + + podSpecPatch: | + imagePullSecrets: + - name: docker-hub-registry + + templates: + # === Top-level DAG === + # Clone repo, then build all 10 variants in parallel + - name: build-matrix + dag: + tasks: + - name: setup + template: setup + + - name: build-variant + template: build-variant + dependencies: [setup] + arguments: + parameters: + - name: triple + value: "{{item.triple}}" + - name: features + value: "{{item.features}}" + - name: cross-image + value: "{{item.cross_image}}" + - name: strip-cmd + value: "{{item.strip_cmd}}" + withItems: + # x86_64-unknown-linux-musl variants + - triple: x86_64-unknown-linux-musl + features: default + cross_image: ghcr.io/cross-rs/x86_64-unknown-linux-musl:main + strip_cmd: strip + - triple: x86_64-unknown-linux-musl + features: full + cross_image: ghcr.io/cross-rs/x86_64-unknown-linux-musl:main + strip_cmd: strip + + # aarch64-unknown-linux-musl variants + - triple: aarch64-unknown-linux-musl + features: default + cross_image: ghcr.io/cross-rs/aarch64-unknown-linux-musl:main + strip_cmd: aarch64-linux-gnu-strip + - triple: aarch64-unknown-linux-musl + features: full + cross_image: ghcr.io/cross-rs/aarch64-unknown-linux-musl:main + strip_cmd: aarch64-linux-gnu-strip + + # x86_64-apple-darwin variants (osxcross) + - triple: x86_64-apple-darwin + features: default + cross_image: ghcr.io/cross-rs/x86_64-apple-darwin:main + strip_cmd: x86_64-apple-darwin-strip + - triple: x86_64-apple-darwin + features: full + cross_image: ghcr.io/cross-rs/x86_64-apple-darwin:main + strip_cmd: x86_64-apple-darwin-strip + + # aarch64-apple-darwin variants (osxcross) + - triple: aarch64-apple-darwin + features: default + cross_image: ghcr.io/cross-rs/aarch64-apple-darwin:main + strip_cmd: aarch64-apple-darwin-strip + - triple: aarch64-apple-darwin + features: full + cross_image: ghcr.io/cross-rs/aarch64-apple-darwin:main + strip_cmd: aarch64-apple-darwin-strip + + # x86_64-pc-windows-gnu variants + - triple: x86_64-pc-windows-gnu + features: default + cross_image: ghcr.io/cross-rs/x86_64-pc-windows-gnu:main + strip_cmd: strip + - triple: x86_64-pc-windows-gnu + features: full + cross_image: ghcr.io/cross-rs/x86_64-pc-windows-gnu:main + strip_cmd: strip + continueOn: + failed: true + + - name: generate-sbom + template: generate-sbom + dependencies: [setup] + + - name: generate-provenance + template: generate-provenance + dependencies: [build-variant] + + # === Setup Step === + # Clone repo and extract version metadata + - name: setup + activeDeadlineSeconds: 600 + container: + image: alpine:3.19 + command: [sh, -c] + args: + - | + set -e + apk add --no-cache git + + git clone --depth 1 --branch "{{workflow.parameters.branch}}" \ + "https://x-access-token:${GH_TOKEN}@github.com/{{workflow.parameters.repo}}.git" \ + /workspace + + cd /workspace + git checkout {{workflow.parameters.commit-sha}} + + # Extract SOURCE_DATE_EPOCH from the tag commit + SOURCE_DATE_EPOCH=$(git show -s --format=%ct {{workflow.parameters.commit-sha}}) + echo "SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}" > /workspace/build-env.sh + + echo "Setup complete for {{workflow.parameters.tag}}" + env: + - name: GH_TOKEN + valueFrom: + secretKeyRef: + name: github-webhook-secret + key: token + volumeMounts: + - name: workspace + mountPath: /workspace + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + + # === Generate CycloneDX SBOM === + # Generates CycloneDX Software Bill of Materials for the workspace + # Produces pdftract-vX.Y.Z.cdx.json covering all Rust dependencies + # + # Bead: pdftract-8zbd + # Plan section: Release Engineering / Artifact Taxonomy, line 3354 + - name: generate-sbom + activeDeadlineSeconds: 600 + container: + image: rust:1.83-bookworm + command: [bash, -c] + args: + - | + set -eo pipefail + + echo "==========================================" + echo "Generating CycloneDX SBOM" + echo "==========================================" + + cd /workspace + export CARGO_HOME="/cache/cargo" + export CARGO_TARGET_DIR="/workspace/target-sbom" + + TAG="{{workflow.parameters.tag}}" + VERSION="{{workflow.parameters.version}}" + SBOM_FILE="pdftract-v${VERSION}.cdx.json" + + # Install cargo-cyclonedx + echo "=== Installing cargo-cyclonedx ===" + cargo install cargo-cyclonedx --locked + + # Generate SBOMs for all workspace members with all dependencies + # Note: cargo-cyclonedx generates separate files for each workspace member + echo "=== Generating SBOMs for workspace ===" + cargo cyclonedx --format json --all --spec-version 1.5 + + # Merge workspace member SBOMs into a single BOM + echo "=== Merging workspace SBOMs ===" + mkdir -p /artifacts + + # Use jq to merge the SBOMs into a single CycloneDX 1.5 BOM + # Each workspace member SBOM is merged, deduplicating components by purl + jq -s ' + { + "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "type": "library", + "lifecycle": "published", + "component": { + "type": "library", + "name": "pdftract", + "version": "'"${VERSION}"'", + "purl": "pkg:cargo/pdftract@" + "'"${VERSION}"'", + "description": "PDF text extraction library with advanced layout analysis", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/jedarden/pdftract.git" + }, + { + "type": "website", + "url": "https://github.com/jedarden/pdftract" + } + ] + } + }, + "components": ([.[].components // []] | flatten | unique_by(.["purl"])) + } + ' \ + crates/pdftract-core/pdftract-core.cdx.json \ + crates/pdftract-cli/pdftract-cli.cdx.json \ + crates/pdftract-py/pdftract-py.cdx.json \ + > "/artifacts/${SBOM_FILE}" + + echo "=== Installing cyclonedx-cli for validation ===" + cargo install cyclonedx-cli --locked + + echo "=== Validating merged SBOM ===" + cyclonedx-cli validate --input-file "/artifacts/${SBOM_FILE}" + + # Display summary + COMPONENT_COUNT=$(jq '.components | length' "/artifacts/${SBOM_FILE}") + echo "=== SBOM generated and validated ===" + echo "Components: ${COMPONENT_COUNT}" + ls -lh "/artifacts/${SBOM_FILE}" + volumeMounts: + - name: workspace + mountPath: /workspace + - name: cargo-cache + mountPath: /cache/cargo + resources: + requests: + cpu: 1000m + memory: 2Gi + limits: + cpu: 2000m + memory: 4Gi + outputs: + artifacts: + - name: sbom + path: /artifacts/pdftract-v{{workflow.parameters.version}}.cdx.json + + # === Build Single Variant === + # Builds one (triple, features) combination, strips, packages, uploads artifact + - name: build-variant + inputs: + parameters: + - name: triple + - name: features + - name: cross-image + - name: strip-cmd + activeDeadlineSeconds: 3600 + retryStrategy: + limit: "1" + retryPolicy: OnError + container: + image: "{{inputs.parameters.cross-image}}" + command: [bash, -c] + args: + - | + set -e + + TRIPLE="{{inputs.parameters.triple}}" + FEATURES="{{inputs.parameters.features}}" + VERSION="{{workflow.parameters.version}}" + STRIP_CMD="{{inputs.parameters.strip-cmd}}" + + # Source build environment (SOURCE_DATE_EPOCH) + if [ -f /workspace/build-env.sh ]; then + . /workspace/build-env.sh + else + SOURCE_DATE_EPOCH=$(git show -s --format=%ct {{workflow.parameters.commit-sha}}) + fi + export SOURCE_DATE_EPOCH + + echo "Building pdftract ${VERSION} for ${TRIPLE} (features: ${FEATURES})" + echo "SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}" + + cd /workspace + + # Install cross if not already present + if ! command -v cross &> /dev/null; then + cargo install cross --locked + fi + + # Build with cross + FEATURE_FLAG="" + if [ "${FEATURES}" != "default" ]; then + FEATURE_FLAG="--features ${FEATURES}" + fi + + cross build --release --target ${TRIPLE} ${FEATURE_FLAG} --locked --frozen + + # Strip the binary + BINARY_PATH="target/${TRIPLE}/release/pdftract" + if [ "${TRIPLE}" = "x86_64-pc-windows-gnu" ]; then + BINARY_PATH="target/${TRIPLE}/release/pdftract.exe" + fi + + # For macOS targets, use osxcross strip from SDK + if [[ "${TRIPLE}" == *-apple-darwin ]]; then + # osxcross strip is mounted at /osxcross/bin + STRIP_PATH="/osxcross/bin/${STRIP_CMD}" + if [ -x "${STRIP_PATH}" ]; then + "${STRIP_PATH}" "${BINARY_PATH}" + else + echo "Warning: osxcross strip not found at ${STRIP_PATH}, binary not stripped" + fi + else + ${STRIP_CMD} "${BINARY_PATH}" + fi + + # Prepare archive layout + ARCHIVE_NAME="pdftract" + if [ "${FEATURES}" != "default" ]; then + ARCHIVE_NAME="pdftract-${FEATURES}" + fi + ARCHIVE_DIR="${ARCHIVE_NAME}-v${VERSION}-${TRIPLE}" + mkdir -p "/artifacts/${ARCHIVE_DIR}" + + # Copy binary + if [ "${TRIPLE}" = "x86_64-pc-windows-gnu" ]; then + cp "${BINARY_PATH}" "/artifacts/${ARCHIVE_DIR}/pdftract.exe" + else + cp "${BINARY_PATH}" "/artifacts/${ARCHIVE_DIR}/pdftract" + fi + + # Copy license files + cp LICENSE-MIT /artifacts/${ARCHIVE_DIR}/ + cp LICENSE-APACHE /artifacts/${ARCHIVE_DIR}/ + cp README.md /artifacts/${ARCHIVE_DIR}/ + + # Extract CHANGELOG excerpt for this version + # Parse CHANGELOG.md for the section under "## [X.Y.Z]" + if [ -f CHANGELOG.md ]; then + python3 <<'EOF' + import re + version = "{{workflow.parameters.version}}".replace(".", r"\.") + with open("CHANGELOG.md", "r") as f: + content = f.read() + # Find the section for this version + pattern = rf"^## \[{version}\](?:.*?)(?=^## |\Z)" + match = re.search(pattern, content, re.MULTILINE | re.DOTALL) + if match: + with open("/artifacts/{{inputs.parameters.triple}}-${{inputs.parameters.features}}/CHANGELOG.md", "w") as out: + out.write(f"## [{version}]\n") + out.write(match.group(0)) + else: + # Fallback: write header only + with open("/artifacts/{{inputs.parameters.triple}}-${{inputs.parameters.features}}/CHANGELOG.md", "w") as out: + out.write(f"## [{version}]\n\nSee https://github.com/{{workflow.parameters.repo}}/releases/tag/v{version}\n") + EOF + else + echo "CHANGELOG.md not found, writing placeholder" + echo "## [${VERSION}]\n\nSee https://github.com/{{workflow.parameters.repo}}/releases/tag/v${VERSION}" \ + > "/artifacts/${ARCHIVE_DIR}/CHANGELOG.md" + fi + + # Create archive + cd /artifacts + if [ "${TRIPLE}" = "x86_64-pc-windows-gnu" ]; then + ARCHIVE_FILE="${ARCHIVE_NAME}-v${VERSION}-${TRIPLE}.zip" + zip -r "${ARCHIVE_FILE}" "${ARCHIVE_DIR}" + else + ARCHIVE_FILE="${ARCHIVE_NAME}-v${VERSION}-${TRIPLE}.tar.gz" + tar czf "${ARCHIVE_FILE}" "${ARCHIVE_DIR}" + fi + + echo "Built archive: ${ARCHIVE_FILE}" + ls -lh "${ARCHIVE_FILE}" + + # Verify size budget for default x86_64-unknown-linux-musl + if [ "${TRIPLE}" = "x86_64-unknown-linux-musl" ] && [ "${FEATURES}" = "default" ]; then + SIZE_BYTES=$(stat -c%s "${BINARY_PATH}" 2>/dev/null || stat -f%z "${BINARY_PATH}") + SIZE_MB=$((SIZE_BYTES / 1024 / 1024)) + echo "Binary size: ${SIZE_MB} MB" + if [ ${SIZE_MB} -gt 4 ]; then + echo "WARNING: Binary exceeds 4 MB budget (${SIZE_MB} MB)" + fi + fi + env: + - name: CARGO_HOME + value: /cache/cargo + - name: CARGO_TARGET_DIR + value: /workspace/target + volumeMounts: + - name: workspace + mountPath: /workspace + - name: cargo-cache + mountPath: /cache/cargo + - name: build-artifacts + mountPath: /artifacts + - name: osxcross-sdk + mountPath: /osxcross + readOnly: true + resources: + requests: + cpu: 2000m + memory: 4Gi + limits: + cpu: 4000m + memory: 8Gi + outputs: + artifacts: + - name: archive + path: /artifacts/*.tar.gz + optional: true + - name: archive-zip + path: /artifacts/*.zip + optional: true + + # === Generate SLSA Provenance === + # Generates multiple.intoto.jsonl with SLSA Provenance v1.0 predicate + # One statement covering all 10 binary archives as subjects + # + # Bead: pdftract-3gk5 + # Plan section: Release Engineering / Signing and Provenance, line 3402 + - name: generate-provenance + activeDeadlineSeconds: 600 + container: + image: python:3.11-slim-bookworm + command: [python3, -c] + args: + - | + import hashlib + import json + import os + import subprocess + from datetime import datetime, timezone + + TAG = "{{workflow.parameters.tag}}" + VERSION = "{{workflow.parameters.version}}" + COMMIT_SHA = "{{workflow.parameters.commit-sha}}" + REPO = "{{workflow.parameters.repo}}" + BUILD_TYPE = "https://argoproj.io/argo-workflows@v1" + BUILDER_ID = "https://iad-ci-oidc.ardenone.com/argo-workflows/pdftract-release-cascade" + + # Artifact directory + artifacts_dir = "/artifacts" + + # Generate SOURCE_DATE_EPOCH for reproducibility + SOURCE_DATE_EPOCH = subprocess.check_output( + ["git", "show", "-s", "--format=%ct", COMMIT_SHA], + cwd="/workspace" + ).decode().strip() + + # Collect all binary archives + subjects = [] + for root, dirs, files in os.walk(artifacts_dir): + for f in files: + if f.endswith((".tar.gz", ".zip")): + path = os.path.join(root, f) + # Compute SHA256 + sha256 = hashlib.sha256() + with open(path, "rb") as fp: + for chunk in iter(lambda: fp.read(65536), b""): + sha256.update(chunk) + digest = sha256.hexdigest() + subjects.append({ + "name": f, + "digest": {"sha256": digest} + }) + print(f"Subject: {f} -> sha256:{digest}") + + # Compute Cargo.lock hash + cargo_lock_path = "/workspace/Cargo.lock" + cargo_lock_hash = None + if os.path.exists(cargo_lock_path): + sha256 = hashlib.sha256() + with open(cargo_lock_path, "rb") as fp: + for chunk in iter(lambda: fp.read(65536), b""): + sha256.update(chunk) + cargo_lock_hash = sha256.hexdigest() + print(f"Cargo.lock: sha256:{cargo_lock_hash}") + + # Build resolved dependencies + resolved_deps = [ + { + "uri": f"git+https://github.com/{REPO}@{COMMIT_SHA}", + "digest": {"gitCommit": COMMIT_SHA} + } + ] + if cargo_lock_hash: + resolved_deps.append({ + "uri": "Cargo.lock", + "digest": {"sha256": cargo_lock_hash} + }) + + # Build invocation ID (reproducible from commit) + invocation_id = f"argo-workflows-{COMMIT_SHA}-{TAG}" + + # Build timestamp from SOURCE_DATE_EPOCH + started_on = datetime.fromtimestamp(int(SOURCE_DATE_EPOCH), tz=timezone.utc).isoformat() + finished_on = datetime.now(timezone.utc).isoformat() + + # Construct SLSA Provenance v1.0 predicate + provenance = { + "_type": "https://in-toto.io/Statement/v1", + "subject": subjects, + "predicateType": "https://slsa.dev/provenance/v1.0", + "predicate": { + "buildDefinition": { + "buildType": BUILD_TYPE, + "externalParameters": { + "tag": TAG, + "version": VERSION, + "commit": COMMIT_SHA, + "source": f"https://github.com/{REPO}" + }, + "internalParameters": { + "workflow": "pdftract-build-binaries", + "targetTriples": [ + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-musl", + "x86_64-apple-darwin", + "aarch64-apple-darwin", + "x86_64-pc-windows-gnu" + ] + }, + "resolvedDependencies": resolved_deps + }, + "runDetails": { + "builder": { + "id": BUILDER_ID, + "builderDependencies": [], + "version": { + "argo": "v3.5.x" + } + }, + "metadata": { + "invocationId": invocation_id, + "startedOn": started_on, + "finishedOn": finished_on + }, + "byproducts": [] + } + } + } + + # Write to multiple.intoto.jsonl (one JSON per line) + output_path = "/provenance/multiple.intoto.jsonl" + os.makedirs(os.path.dirname(output_path), exist_ok=True) + with open(output_path, "w") as f: + f.write(json.dumps(provenance) + "\n") + + print(f"Provenance written to {output_path}") + print(f"Subjects: {len(subjects)}") + print(f"Invocation ID: {invocation_id}") + volumeMounts: + - name: build-artifacts + mountPath: /artifacts + - name: workspace + mountPath: /workspace + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + outputs: + artifacts: + - name: provenance + path: /provenance/multiple.intoto.jsonl + + # === Collect Provenance Output === + # Helper template to aggregate provenance from generate-provenance + # This ensures the provenance file is available as a workflow-level output + - name: collect-provenance-output + activeDeadlineSeconds: 60 + container: + image: alpine:3.19 + command: [sh, -c] + args: + - | + set -e + echo "=== Collecting provenance artifacts ===" + # Ensure output directory exists + mkdir -p /output/provenance + # Copy provenance if exists + if [ -f /provenance/multiple.intoto.jsonl ]; then + cp /provenance/multiple.intoto.jsonl /output/provenance/ + echo "Provenance collected: multiple.intoto.jsonl" + ls -lh /output/provenance/ + else + echo "Warning: provenance file not found" + fi + volumeMounts: + - name: build-artifacts + mountPath: /artifacts + - name: workspace + mountPath: /workspace + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + outputs: + artifacts: + - name: provenance-output + path: /output/provenance/multiple.intoto.jsonl + optional: true diff --git a/.ci/argo-workflows/pdftract-crates-publish.yaml b/.ci/argo-workflows/pdftract-crates-publish.yaml new file mode 100644 index 0000000..ec625cd --- /dev/null +++ b/.ci/argo-workflows/pdftract-crates-publish.yaml @@ -0,0 +1,283 @@ +# pdftract-crates-publish WorkflowTemplate +# +# Publishes pdftract-core, pdftract-cli, and optionally pdftract-libpdftract to crates.io. +# Triggered by milestone tag after pdftract-build-binaries completes successfully. +# +# === Critical Ordering Requirement === +# crates.io has a hard ordering requirement: a crate cannot depend on a sibling whose +# version is not yet visible in the registry index. This template enforces the ordering: +# 1. publish-core: Publish pdftract-core to crates.io +# 2. wait-index-core: Poll crates.io sparse index until core version appears +# 3. publish-cli: Publish pdftract-cli (depends on pdftract-core) +# 4. wait-index-cli: Poll crates.io sparse index until cli version appears +# 5. publish-libpdftract: (optional) Publish pdftract-libpdftract +# +# The index-propagation wait CANNOT be skipped — crates.io's sparse index has up to +# 60 seconds of propagation delay even on green API responses. Without the wait, +# pdftract-cli publish would fail with "no matching version" errors. +# +# === Idempotency === +# crates.io rejects duplicate publishes. The publish step treats "already uploaded" +# errors as success, making re-runs after partial failure safe. +# +# === Partial Failure Recovery === +# If a publish run fails partway through, the half-published version can be yanked: +# cargo yank --vers X.Y.Z pdftract-core +# Then re-run the workflow. +# +# === crates.io API Token === +# Uses token from ExternalSecret `crates-io-token-pdftract` (synced from OpenBao). +# Token rotation: every 90 days per docs/operations/secrets-rotation.md. +# +# Bead: pdftract-5x3u +# Plan section: Release Engineering / Argo WorkflowTemplates, line 3391 +# +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: pdftract-crates-publish + namespace: argo-workflows + labels: + app.kubernetes.io/name: pdftract-crates-publish + app.kubernetes.io/component: ci + app.kubernetes.io/part-of: pdftract +spec: + entrypoint: publish-crates + serviceAccountName: argo-workflow + + podGC: + strategy: OnPodCompletion + + ttlStrategy: + secondsAfterSuccess: 1800 + secondsAfterFailure: 7200 + + arguments: + parameters: + - name: tag + value: "" + description: "Git tag triggering the publish (e.g., v0.3.0)" + - name: dry_run + value: "false" + description: "Set to 'true' for testing -- skips actual publish" + - name: publish_libpdftract + value: "false" + description: "Set to 'true' to also publish pdftract-libpdftract after cli" + + volumes: + - name: cargo-home + emptyDir: {} + - name: cargo-registry + emptyDir: {} + + podMetadata: + labels: + app.kubernetes.io/name: pdftract-crates-publish + tag: "{{workflow.parameters.tag}}" + templates: + - name: publish-crates + dag: + tasks: + - name: publish-core + template: publish-crate + arguments: + parameters: + - name: crate + value: pdftract-core + - name: tag + value: "{{workflow.parameters.tag}}" + - name: dry_run + value: "{{workflow.parameters.dry_run}}" + + - name: wait-index-core + template: wait-crates-index + dependencies: [publish-core] + arguments: + parameters: + - name: crate + value: pdftract-core + - name: tag + value: "{{workflow.parameters.tag}}" + + - name: publish-cli + template: publish-crate + dependencies: [wait-index-core] + arguments: + parameters: + - name: crate + value: pdftract-cli + - name: tag + value: "{{workflow.parameters.tag}}" + - name: dry_run + value: "{{workflow.parameters.dry_run}}" + + - name: wait-index-cli + template: wait-crates-index + dependencies: [publish-cli] + arguments: + parameters: + - name: crate + value: pdftract-cli + - name: tag + value: "{{workflow.parameters.tag}}" + + - name: publish-libpdftract + template: publish-crate + when: "{{workflow.parameters.publish_libpdftract}} == true" + dependencies: [wait-index-cli] + arguments: + parameters: + - name: crate + value: pdftract-libpdftract + - name: tag + value: "{{workflow.parameters.tag}}" + - name: dry_run + value: "{{workflow.parameters.dry_run}}" + + - name: publish-crate + inputs: + parameters: + - name: crate + - name: tag + - name: dry_run + retryStrategy: + limit: 2 + retryPolicy: OnError + script: + image: rust:1.78-slim + command: [bash] + source: | + set -e + + CRATE="{{inputs.parameters.crate}}" + TAG="{{inputs.parameters.tag}}" + DRY_RUN="{{inputs.parameters.dry_run}}" + + # Clone repo at the tag + git clone --depth 1 --branch "$TAG" \ + "https://github.com/jedarden/pdftract.git" /workspace + cd /workspace + + # Install cargo config with credentials + mkdir -p "$CARGO_HOME/registry" + cat > "$CARGO_HOME/config" <<'EOF' + [registry] + default = "crates-io" + + [http] + timeout = 60 + EOF + + # Extract version from tag (strip 'v' prefix if present) + VERSION="${TAG#v}" + echo "Publishing $CRATE version $VERSION" + + # Verify version matches Cargo.toml + CARGO_VERSION=$(grep -A10 "^\[package\]" crates/$CRATE/Cargo.toml 2>/dev/null | grep '^version' | sed 's/.*= *"\(.*\)".*/\1/') + if [ "$CRATE" = "pdftract-core" ]; then + CARGO_VERSION=$(grep -A10 "^\[package\]" Cargo.toml | grep '^version' | sed 's/.*= *"\(.*\)".*/\1/') + elif [ "$CRATE" = "pdftract-cli" ]; then + CARGO_VERSION=$(grep -A10 "^\[package\]" crates/pdftract-cli/Cargo.toml | grep '^version' | sed 's/.*= *"\(.*\)".*/\1/') + elif [ "$CRATE" = "pdftract-libpdftract" ]; then + CARGO_VERSION=$(grep -A10 "^\[package\]" crates/pdftract-libpdftract/Cargo.toml 2>/dev/null | grep '^version' | sed 's/.*= *"\(.*\)".*/\1/') + fi + + if [ "$VERSION" != "$CARGO_VERSION" ]; then + echo "Error: Tag version $VERSION does not match Cargo.toml version $CARGO_VERSION" + exit 1 + fi + + # Build dry-run or publish command + if [ "$DRY_RUN" = "true" ]; then + echo "[DRY RUN] Would publish $CRATE $VERSION" + cargo publish -p "$CRATE" --locked --token "$CARGO_REGISTRY_TOKEN" --dry-run + exit 0 + fi + + # Attempt publish; treat "already uploaded" as success (idempotent) + if cargo publish -p "$CRATE" --locked --token "$CARGO_REGISTRY_TOKEN" 2>&1 | tee /tmp/publish.log; then + echo "Published $CRATE $VERSION successfully" + else + EXIT_CODE=$? + if grep -q "already uploaded" /tmp/publish.log; then + echo "$CRATE $VERSION is already published (idempotent — continuing)" + exit 0 + else + echo "Publish failed with exit code $EXIT_CODE" + cat /tmp/publish.log + exit $EXIT_CODE + fi + fi + env: + - name: CARGO_HOME + value: /cargo + - name: CARGO_REGISTRY_TOKEN + valueFrom: + secretKeyRef: + name: crates-io-token-pdftract + key: token + volumeMounts: + - name: cargo-home + mountPath: /cargo + - name: cargo-registry + mountPath: /cargo/registry + resources: + requests: + cpu: 1000m + memory: 2Gi + limits: + cpu: 2000m + memory: 4Gi + activeDeadlineSeconds: 600 + + - name: wait-crates-index + inputs: + parameters: + - name: crate + - name: tag + retryStrategy: + limit: 1 + script: + image: alpine:3.19 + command: [sh] + source: | + set -e + + CRATE="{{inputs.parameters.crate}}" + TAG="{{inputs.parameters.tag}}" + VERSION="${TAG#v}" + + # crates.io sparse index path: first 2 chars of reversed crate name, then next 2 + # e.g. pdftract-core -> er-octfpdf -> core-pdftract + CRATE_REVERSED=$(echo "$CRATE" | rev) + INDEX_PATH="${CRATE_REVERSED:0:2}/${CRATE_REVERSED:2:2}/$CRATE" + + echo "Waiting for $CRATE $VERSION to appear in crates.io index..." + echo "Index path: $INDEX_PATH" + + TIMEOUT=300 + INTERVAL=10 + ELAPSED=0 + + while [ $ELAPSED -lt $TIMEOUT ]; do + if curl -fsS "https://index.crates.io/$INDEX_PATH" 2>/dev/null | grep -q "\"$VERSION\""; then + echo "Version $VERSION found in crates.io index after ${ELAPSED}s" + exit 0 + fi + + echo "Not yet indexed (${ELAPSED}s)... waiting ${INTERVAL}s" + sleep $INTERVAL + ELAPSED=$((ELAPSED + INTERVAL)) + done + + echo "ERROR: Timeout after ${TIMEOUT}s — version $VERSION not found in crates.io index" + echo "Recovery: cargo yank --vers $VERSION $CRATE" + exit 1 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + activeDeadlineSeconds: 360 diff --git a/.ci/argo-workflows/pdftract-docker-build.yaml b/.ci/argo-workflows/pdftract-docker-build.yaml new file mode 100644 index 0000000..26ea06f --- /dev/null +++ b/.ci/argo-workflows/pdftract-docker-build.yaml @@ -0,0 +1,713 @@ +# pdftract-docker-build WorkflowTemplate +# +# Builds 3 multi-arch Docker images (latest, ocr, full) for amd64 + arm64 +# Triggered by milestone tag (vX.Y.Z). Pushes to GHCR and signs with cosign keyless. +# +# Image variants: +# - latest: default features, distroless base (~20 MB) +# - ocr: default + OCR features, debian-slim with Tesseract (~120 MB) +# - full: all features, debian-slim with Tesseract (~140 MB) +# +# Bead: pdftract-68pe +# Plan section: Release Engineering / Argo WorkflowTemplates, line 3392 +# +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: pdftract-docker-build + namespace: argo-workflows + labels: + app.kubernetes.io/name: pdftract-docker-build + app.kubernetes.io/component: ci + app.kubernetes.io/part-of: pdftract +spec: + entrypoint: build-all-variants + serviceAccountName: argo-workflow + + podGC: OnPodCompletion + + ttlSecondsAfterFinished: + success: 1800 + failure: 7200 + + arguments: + parameters: + - name: repo + value: jedarden/pdftract + - name: branch + value: main + - name: tag + value: "" + description: "Git tag (e.g., v0.1.0)" + - name: version + value: "" + description: "Version extracted from tag (e.g., 0.1.0)" + - name: commit-sha + value: "" + description: "Full commit SHA of the tag" + + volumeClaimTemplates: + - metadata: + name: workspace + spec: + accessModes: [ReadWriteOnce] + storageClassName: sata-large + resources: + requests: + storage: 10Gi + - metadata: + name: cargo-cache + spec: + accessModes: [ReadWriteOnce] + storageClassName: sata-large + resources: + requests: + storage: 50Gi + + volumes: + - name: docker-config + secret: + secretName: ghcr-registry + items: + - key: .dockerconfigjson + path: config.json + # Projected service account token for OIDC-based cosign keyless signing + - name: oidc-token + projected: + sources: + - serviceAccountToken: + path: token + audience: sigstore + expirationSeconds: 600 + + podMetadata: + labels: + app.kubernetes.io/name: pdftract-docker-build + tag: "{{workflow.parameters.tag}}" + + podSpecPatch: | + imagePullSecrets: + - name: ghcr-registry + + templates: + # === Top-level DAG === + # Build all 3 image variants in parallel, then sign each + - name: build-all-variants + dag: + tasks: + - name: setup + template: setup + + - name: build-latest + template: build-multi-arch + dependencies: [setup] + arguments: + parameters: + - name: variant + value: "latest" + - name: features + value: "default" + + - name: build-ocr + template: build-multi-arch + dependencies: [setup] + arguments: + parameters: + - name: variant + value: "ocr" + - name: features + value: "ocr" + + - name: build-full + template: build-multi-arch + dependencies: [setup] + arguments: + parameters: + - name: variant + value: "full" + - name: features + value: "full" + + - name: sign-latest + template: sign-image + dependencies: [build-latest] + arguments: + parameters: + - name: variant + value: "latest" + - name: digest + value: "{{tasks.build-latest.outputs.parameters.digest}}" + + - name: sign-ocr + template: sign-image + dependencies: [build-ocr] + arguments: + parameters: + - name: variant + value: "ocr" + - name: digest + value: "{{tasks.build-ocr.outputs.parameters.digest}}" + + - name: sign-full + template: sign-image + dependencies: [build-full] + arguments: + parameters: + - name: variant + value: "full" + - name: digest + value: "{{tasks.build-full.outputs.parameters.digest}}" + + - name: generate-sbom + template: generate-sbom + dependencies: [setup] + + - name: attest-sbom-latest + template: attest-sbom + dependencies: [sign-latest, generate-sbom] + arguments: + parameters: + - name: variant + value: "latest" + - name: digest + value: "{{tasks.build-latest.outputs.parameters.digest}}" + artifacts: + - name: sbom + from: "{{tasks.generate-sbom.outputs.artifacts.sbom}}" + + - name: attest-sbom-ocr + template: attest-sbom + dependencies: [sign-ocr, generate-sbom] + arguments: + parameters: + - name: variant + value: "ocr" + - name: digest + value: "{{tasks.build-ocr.outputs.parameters.digest}}" + artifacts: + - name: sbom + from: "{{tasks.generate-sbom.outputs.artifacts.sbom}}" + + - name: attest-sbom-full + template: attest-sbom + dependencies: [sign-full, generate-sbom] + arguments: + parameters: + - name: variant + value: "full" + - name: digest + value: "{{tasks.build-full.outputs.parameters.digest}}" + artifacts: + - name: sbom + from: "{{tasks.generate-sbom.outputs.artifacts.sbom}}" + + # === Setup Step === + # Clone repo and prepare workspace + - name: setup + activeDeadlineSeconds: 600 + container: + image: alpine:3.19 + command: [sh, -c] + args: + - | + set -e + apk add --no-cache git + + git clone --depth 1 --branch "{{workflow.parameters.branch}}" \ + "https://x-access-token:${GH_TOKEN}@github.com/{{workflow.parameters.repo}}.git" \ + /workspace + + cd /workspace + git checkout {{workflow.parameters.commit-sha}} + + echo "Setup complete for {{workflow.parameters.tag}}" + echo "Version: {{workflow.parameters.version}}" + env: + - name: GH_TOKEN + valueFrom: + secretKeyRef: + name: github-pat-pdftract + key: token + volumeMounts: + - name: workspace + mountPath: /workspace + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + + # === Generate CycloneDX SBOM === + # Generates CycloneDX Software Bill of Materials for the workspace + # Produces pdftract-vX.Y.Z.cdx.json covering all Rust dependencies + # + # Bead: pdftract-8zbd + # Plan section: Release Engineering / Artifact Taxonomy, line 3354 + - name: generate-sbom + activeDeadlineSeconds: 600 + container: + image: rust:1.83-bookworm + command: [bash, -c] + args: + - | + set -eo pipefail + + echo "==========================================" + echo "Generating CycloneDX SBOM" + echo "==========================================" + + cd /workspace + export CARGO_HOME="/cache/cargo" + export CARGO_TARGET_DIR="/workspace/target-sbom" + + TAG="{{workflow.parameters.tag}}" + VERSION="{{workflow.parameters.version}}" + SBOM_FILE="pdftract-v${VERSION}.cdx.json" + + # Install cargo-cyclonedx + echo "=== Installing cargo-cyclonedx ===" + cargo install cargo-cyclonedx --locked + + # Generate SBOMs for all workspace members with all dependencies + # Note: cargo-cyclonedx generates separate files for each workspace member + echo "=== Generating SBOMs for workspace ===" + cargo cyclonedx --format json --all --spec-version 1.5 + + # Merge workspace member SBOMs into a single BOM + echo "=== Merging workspace SBOMs ===" + mkdir -p /sbom + + # Use jq to merge the SBOMs into a single CycloneDX 1.5 BOM + # Each workspace member SBOM is merged, deduplicating components by purl + jq -s ' + { + "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "type": "library", + "lifecycle": "published", + "component": { + "type": "library", + "name": "pdftract", + "version": "'"${VERSION}"'", + "purl": "pkg:cargo/pdftract@" + "'"${VERSION}"'", + "description": "PDF text extraction library with advanced layout analysis", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/jedarden/pdftract.git" + }, + { + "type": "website", + "url": "https://github.com/jedarden/pdftract" + } + ] + } + }, + "components": ([.[].components // []] | flatten | unique_by(.["purl"])) + } + ' \ + crates/pdftract-core/pdftract-core.cdx.json \ + crates/pdftract-cli/pdftract-cli.cdx.json \ + crates/pdftract-py/pdftract-py.cdx.json \ + > "/sbom/${SBOM_FILE}" + + # Validate the SBOM + echo "=== Installing cyclonedx-cli for validation ===" + cargo install cyclonedx-cli --locked + + echo "=== Validating merged SBOM ===" + cyclonedx-cli validate --input-file "/sbom/${SBOM_FILE}" + + # Display summary + COMPONENT_COUNT=$(jq '.components | length' "/sbom/${SBOM_FILE}") + echo "=== SBOM generated and validated ===" + echo "Components: ${COMPONENT_COUNT}" + ls -lh "/sbom/${SBOM_FILE}" + volumeMounts: + - name: workspace + mountPath: /workspace + - name: cargo-cache + mountPath: /cache/cargo + resources: + requests: + cpu: 1000m + memory: 2Gi + limits: + cpu: 2000m + memory: 4Gi + outputs: + artifacts: + - name: sbom + path: /sbom/pdftract-v{{workflow.parameters.version}}.cdx.json + archiveNone: false + + # === Build Multi-Arch Image === + # Uses docker buildx to build and push multi-arch manifest lists + - name: build-multi-arch + inputs: + parameters: + - name: variant + description: "Image variant: latest, ocr, or full" + - name: features + description: "Cargo features to build" + activeDeadlineSeconds: 2400 + retryStrategy: + limit: "2" + retryPolicy: OnError + backoff: + duration: 60s + factor: "2" + container: + image: docker:dind + command: [sh, -c] + args: + - | + set -e + + VARIANT="{{inputs.parameters.variant}}" + FEATURES="{{inputs.parameters.features}}" + VERSION="{{workflow.parameters.version}}" + TAG="{{workflow.parameters.tag}}" + REGISTRY="ghcr.io/jedarden/pdftract" + + echo "=== Building pdftract:${VARIANT} (features: ${FEATURES}) ===" + + # Start Docker daemon + dockerd-entrypoint.sh & + sleep 5 + + # Configure Docker for GHCR auth + mkdir -p /root/.docker + cp /root/.dockerconfigjson /root/.docker/config.json + + # Create buildx builder with QEMU support + docker buildx create --use --name multiarch-builder --driver docker-container --bootstrap + + # Enable QEMU for cross-platform emulation + docker buildx inspect --bootstrap + + cd /workspace + + # Build and push multi-arch image + # Tag format: ghcr.io/jedarden/pdftract:X.Y.Z and :latest (or :ocr, :full) + VERSION_TAG="${REGISTRY}:${TAG}" + if [ "${VARIANT}" = "latest" ]; then + FLOATING_TAG="${REGISTRY}:latest" + elif [ "${VARIANT}" = "ocr" ]; then + VERSION_TAG="${REGISTRY}:ocr-${TAG}" + FLOATING_TAG="${REGISTRY}:ocr" + else + VERSION_TAG="${REGISTRY}:full-${TAG}" + FLOATING_TAG="${REGISTRY}:full" + fi + + # Build and push with buildx + # Buildx creates a multi-arch manifest list + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --build-arg FEATURES=${FEATURES} \ + --tag "${VERSION_TAG}" \ + --tag "${FLOATING_TAG}" \ + --file Dockerfile \ + --push \ + --progress=plain \ + . + + echo "=== Build complete ===" + + # Get the manifest digest for signing + # Use docker buildx imagetools inspect with JSON output for reliable parsing + INSPECT_OUTPUT=$(docker buildx imagetools inspect "${VERSION_TAG}" --format '{{json .}}' 2>/dev/null || echo '{}') + + # Extract digest from JSON - the manifest digest for the multi-arch image + DIGEST=$(echo "${INSPECT_OUTPUT}" | grep -o '"manifestDigest":"[^"]*"' | cut -d'"' -f4 || echo "") + + # Fallback: try crane if available (more reliable for digest extraction) + if [ -z "${DIGEST}" ]; then + if command -v crane >/dev/null 2>&1; then + DIGEST=$(crane digest "${VERSION_TAG}" 2>/dev/null || echo "") + fi + fi + + # Final fallback: use docker manifest inspect + if [ -z "${DIGEST}" ]; then + DIGEST=$(docker manifest inspect "${VERSION_TAG}" 2>/dev/null | \ + grep -o '"sha256:[a-f0-9]*"' | head -1 | tr -d '"' || echo "") + fi + + echo "Digest: ${DIGEST}" + echo "${DIGEST}" > /tmp/digest + + # Verify the image was pushed + echo "=== Verifying pushed image ===" + docker buildx imagetools inspect "${VERSION_TAG}" + env: + - name: DOCKER_HOST + value: tcp://127.0.0.1:2375 + - name: DOCKER_TLS_CERTDIR + value: "" + volumeMounts: + - name: workspace + mountPath: /workspace + - name: docker-config + mountPath: /root/.dockerconfigjson + subPath: config.json + resources: + requests: + cpu: 2000m + memory: 4Gi + limits: + cpu: 8000m + memory: 16Gi + securityContext: + privileged: true + outputs: + parameters: + - name: digest + valueFrom: + path: /tmp/digest + + # === Sign Image with Cosign === + # Signs the multi-arch manifest list using keyless OIDC + # and attaches SLSA provenance attestation + - name: sign-image + inputs: + parameters: + - name: variant + description: "Image variant: latest, ocr, or full" + - name: digest + description: "Manifest digest to sign" + activeDeadlineSeconds: 600 + retryStrategy: + limit: "3" + retryPolicy: OnError + backoff: + duration: 30s + factor: "2" + container: + image: ghcr.io/sigstore/cosign:v2.2.3 + command: [sh, -c] + args: + - | + set -e + + VARIANT="{{inputs.parameters.variant}}" + DIGEST="{{inputs.parameters.digest}}" + TAG="{{workflow.parameters.tag}}" + VERSION="{{workflow.parameters.version}}" + REGISTRY="ghcr.io/jedarden/pdftract" + COMMIT="{{workflow.parameters.commit-sha}}" + + echo "=== Signing pdftract:${VARIANT} ===" + + # Construct image reference with digest + if [ "${VARIANT}" = "latest" ]; then + IMAGE_REF="${REGISTRY}:${TAG}@${DIGEST}" + elif [ "${VARIANT}" = "ocr" ]; then + IMAGE_REF="${REGISTRY}:ocr-${TAG}@${DIGEST}" + else + IMAGE_REF="${REGISTRY}:full-${TAG}@${DIGEST}" + fi + + echo "Signing: ${IMAGE_REF}" + + # Sign with cosign (keyless, OIDC from iad-ci) + # The OIDC token is mounted from the projected service account token volume + cosign sign \ + --yes \ + --oidc-issuer-url="${COSIGN_OIDC_ISSUER}" \ + "${IMAGE_REF}" + + echo "=== Signature created ===" + + # Verify the signature (sanity check with certificate identity) + echo "=== Verifying signature (sanity check) ===" + cosign verify \ + --certificate-identity-regexp "${COSIGN_CERTIFICATE_IDENTITY}" \ + --certificate-oidc-issuer "${COSIGN_OIDC_ISSUER}" \ + "${IMAGE_REF}" || echo "Warning: Verification failed in-cluster (expected if OIDC issuer not reachable from pod)" + + # Generate SLSA provenance attestation + echo "=== Generating SLSA v1.0 provenance attestation ===" + + # Create SLSA Provenance v1.0 predicate + # Format: https://slsa.dev/spec/v1.0/provenance + cat > /tmp/provenance.json </dev/null 2>&1 + + # Clone repo at the tag + git clone --depth 1 --branch "$TAG" \ + "https://github.com/jedarden/pdftract.git" /workspace + cd /workspace + + # Install mdBook tools (cached in cargo home for speed) + echo "Installing mdBook and mdbook-linkcheck..." + cargo install mdbook mdbook-linkcheck --locked + + # Verify book.toml exists + if [ ! -f "docs/user-docs/book.toml" ]; then + echo "Error: docs/user-docs/book.toml not found" + exit 1 + fi + + # Run linkcheck + echo "Running mdbook-linkcheck..." + # mdbook-linkcheck returns 2 for warnings, 1 for errors + # We want to fail on errors (internal links) but allow warnings (external links) + LINKCHECK_EXIT=0 + cd docs/user-docs + mdbook-linkcheck . || LINKCHECK_EXIT=$? + + if [ "$LINKCHECK_EXIT" -eq 1 ]; then + echo "Error: mdbook-linkcheck found broken internal links" + exit 1 + elif [ "$LINKCHECK_EXIT" -eq 2 ]; then + echo "Warning: mdbook-linkcheck found issues with external links (continuing)" + fi + + # Build mdBook (respects book.toml build-dir = "build/user-docs") + echo "Building mdBook..." + mdbook build + + # Verify build output + if [ ! -f "build/user-docs/index.html" ]; then + echo "Error: mdBook build failed - index.html not found" + exit 1 + fi + + # Verify schema rendering present + if [ ! -d "build/user-docs/schema" ]; then + echo "Error: Schema docs missing from build output" + exit 1 + fi + + # Install wrangler for Cloudflare Pages deploy + echo "Installing wrangler..." + npm install -g wrangler@latest + + # Deploy to Cloudflare Pages (production environment at pdftract.com) + echo "Deploying to Cloudflare Pages (pdftract.com)..." + wrangler pages deploy build/user-docs \ + --project-name=pdftract-com \ + --branch="$TAG" \ + --commit-dirty=true + + echo "Docs deployed successfully to pdftract.com for tag $TAG" + env: + - name: CARGO_HOME + value: /cargo + - name: CLOUDFLARE_API_TOKEN + valueFrom: + secretKeyRef: + name: cloudflare-pages-secret + key: CF_API_TOKEN + - name: CLOUDFLARE_ACCOUNT_ID + value: "e26f015c7ba47a6ad6219385e77072b7" + volumeMounts: + - name: cargo-home + mountPath: /cargo + - name: cargo-registry + mountPath: /cargo/registry + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 2000m + memory: 4Gi + activeDeadlineSeconds: 1200 diff --git a/.ci/argo-workflows/pdftract-github-release.yaml b/.ci/argo-workflows/pdftract-github-release.yaml new file mode 100644 index 0000000..96f76e6 --- /dev/null +++ b/.ci/argo-workflows/pdftract-github-release.yaml @@ -0,0 +1,1268 @@ +# pdftract-github-release WorkflowTemplate +# +# Orchestrates the final GitHub Release creation for a milestone tag. +# Runs after pdftract-build-binaries, pdftract-py-ci, pdftract-crates-publish, +# and pdftract-docker-build complete. Collects all release artifacts, computes +# the aggregate SHA256SUMS file, signs it with cosign (keyless OIDC), generates +# release notes via git-cliff, and creates the GitHub Release with all artifacts. +# +# Artifacts attached to the release: +# - 10 binary archives (5 triples × 2 feature variants) +# - 5 Python wheels + 1 sdist +# - SHA256SUMS (aggregate checksum) +# - SHA256SUMS.sig (cosign signature) +# - multiple.intoto.jsonl (SLSA L3 provenance) +# - pdftract-vX.Y.Z.cdx.json (CycloneDX SBOM) +# +# Pre-release tags (vX.Y.Z-rc.N) are marked as pre-release. +# +# Bead: pdftract-2x7y +# Plan section: Release Engineering / Argo WorkflowTemplates, line 3393 +# +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: pdftract-github-release + namespace: argo-workflows + labels: + app.kubernetes.io/name: pdftract-github-release + app.kubernetes.io/component: ci + app.kubernetes.io/part-of: pdftract +spec: + entrypoint: release-pipeline + serviceAccountName: argo-workflow + + podGC: OnPodCompletion + + ttlSecondsAfterFinished: + success: 1800 + failure: 7200 + + arguments: + parameters: + - name: repo + value: jedarden/pdftract + - name: branch + value: main + - name: tag + value: "" + description: "Git tag (e.g., v0.1.0)" + - name: version + value: "" + description: "Version extracted from tag (e.g., 0.1.0)" + - name: commit-sha + value: "" + description: "Full commit SHA of the tag" + - name: parent-workflow + value: "" + description: "Parent workflow name for retrieving artifacts from upstream workflows" + - name: slsa-artifact-exists + value: "false" + description: "Whether SLSA provenance file is available (true/false)" + - name: sbom-artifact-exists + value: "false" + description: "Whether SBOM file is available (true/false)" + + volumeClaimTemplates: + - metadata: + name: release-artifacts + spec: + accessModes: [ReadWriteOnce] + storageClassName: sata-large + resources: + requests: + storage: 10Gi + - metadata: + name: workspace + spec: + accessModes: [ReadWriteOnce] + storageClassName: sata-large + resources: + requests: + storage: 5Gi + + volumes: + - name: docker-config + secret: + secretName: docker-hub-registry + items: + - key: .dockerconfigjson + path: config.json + # Projected service account token for OIDC-based cosign keyless signing + - name: oidc-token + projected: + sources: + - serviceAccountToken: + path: token + audience: sigstore + expirationSeconds: 600 + + podMetadata: + labels: + app.kubernetes.io/name: pdftract-github-release + tag: "{{workflow.parameters.tag}}" + + podSpecPatch: | + imagePullSecrets: + - name: docker-hub-registry + + templates: + # === Top-level DAG === + # Clone repo, collect artifacts, compute checksums, sign, generate notes, create release + # + # NOTE: When invoked from a cascade workflow, the upstream workflows should + # pass their artifacts as inputs to the 'collect-artifacts' task using + # `artifacts: - name: binary-archives from: /build-variant:archive` + # The cascade workflow handles the artifact collection and passes them here. + - name: release-pipeline + dag: + tasks: + - name: setup + template: setup + + - name: collect-artifacts + template: collect-artifacts + dependencies: [setup] + arguments: + artifacts: + # Binary archives from pdftract-build-binaries (10 total) + # These should be passed from the upstream workflow when invoked via cascade + - name: binary-archives + # In cascade mode, this is populated from upstream workflow outputs + # For standalone invocation, artifacts are downloaded from GitHub + optional: true + + # Python packages from pdftract-py-ci (6 total: 5 wheels + 1 sdist) + - name: python-packages + optional: true + + # SLSA provenance from upstream build + - name: slsa-provenance + optional: true + + # CycloneDX SBOM from upstream build + - name: cyclonedx-sbom + optional: true + + - name: retrieve-provenance + template: retrieve-provenance + dependencies: [setup] + + - name: generate-sbom + template: generate-sbom + dependencies: [setup] + + - name: compute-sha256sums + template: compute-sha256sums + dependencies: [collect-artifacts, generate-sbom] + + - name: sign-sums + template: sign-sums + dependencies: [compute-sha256sums] + + - name: verify-provenance + template: verify-provenance + dependencies: [retrieve-provenance] + + - name: git-cliff-notes + template: git-cliff-notes + dependencies: [setup] + + - name: gh-release-create + template: gh-release-create + dependencies: [sign-sums, verify-provenance, git-cliff-notes, generate-sbom] + arguments: + parameters: + - name: is-prerelease + value: | + {{- if regexMatch "^v[0-9]+\\.[0-9]+\\.[0-9]+[-+]" workflow.parameters.tag -}} + true + {{- else -}} + false + {{- end -}} + artifacts: + - name: release-notes + from: "{{tasks.git-cliff-notes.outputs.artifacts.release-notes}}" + - name: sha256sums + from: "{{tasks.compute-sha256sums.outputs.artifacts.sha256sums}}" + - name: sha256sums-sig + from: "{{tasks.sign-sums.outputs.artifacts.sha256sums-sig}}" + - name: collected-artifacts + from: "{{tasks.collect-artifacts.outputs.artifacts.collected-artifacts}}" + - name: retrieved-provenance + from: "{{tasks.retrieve-provenance.outputs.artifacts.retrieved-provenance}}" + - name: generated-sbom + from: "{{tasks.generate-sbom.outputs.artifacts.sbom}}" + + # === Setup Step === + # Clone repo and prepare workspace + - name: setup + activeDeadlineSeconds: 600 + container: + image: alpine:3.19 + command: [sh, -c] + args: + - | + set -e + apk add --no-cache git + + git clone --depth 1 --branch "{{workflow.parameters.branch}}" \ + "https://x-access-token:${GH_TOKEN}@github.com/{{workflow.parameters.repo}}.git" \ + /workspace + + cd /workspace + git checkout {{workflow.parameters.commit-sha}} + + echo "Setup complete for {{workflow.parameters.tag}}" + env: + - name: GH_TOKEN + valueFrom: + secretKeyRef: + name: github-pat-pdftract + key: token + volumeMounts: + - name: workspace + mountPath: /workspace + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + + # === Generate CycloneDX SBOM === + # Generates CycloneDX Software Bill of Materials for the workspace + # Produces pdftract-vX.Y.Z.cdx.json covering all Rust dependencies + # + # Bead: pdftract-8zbd + # Plan section: Release Engineering / Artifact Taxonomy, line 3354 + - name: generate-sbom + activeDeadlineSeconds: 600 + container: + image: rust:1.83-bookworm + command: [bash, -c] + args: + - | + set -eo pipefail + + echo "==========================================" + echo "Generating CycloneDX SBOM" + echo "==========================================" + + cd /workspace + export CARGO_HOME="/cache/cargo" + export CARGO_TARGET_DIR="/workspace/target-sbom" + + TAG="{{workflow.parameters.tag}}" + VERSION="{{workflow.parameters.version}}" + SBOM_FILE="pdftract-v${VERSION}.cdx.json" + + # Install cargo-cyclonedx + echo "=== Installing cargo-cyclonedx ===" + cargo install cargo-cyclonedx --locked + + # Generate SBOMs for all workspace members with all dependencies + # Note: cargo-cyclonedx generates separate files for each workspace member + echo "=== Generating SBOMs for workspace ===" + cargo cyclonedx --format json --all --spec-version 1.5 + + # Merge workspace member SBOMs into a single BOM + echo "=== Merging workspace SBOMs ===" + mkdir -p /sbom + + # Use jq to merge the SBOMs into a single CycloneDX 1.5 BOM + # Each workspace member SBOM is merged, deduplicating components by purl + jq -s ' + { + "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "type": "library", + "lifecycle": "published", + "component": { + "type": "library", + "name": "pdftract", + "version": "'"${VERSION}"'", + "purl": "pkg:cargo/pdftract@" + "'"${VERSION}"'", + "description": "PDF text extraction library with advanced layout analysis", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/jedarden/pdftract.git" + }, + { + "type": "website", + "url": "https://github.com/jedarden/pdftract" + } + ] + } + }, + "components": ([.[].components // []] | flatten | unique_by(.["purl"])) + } + ' \ + crates/pdftract-core/pdftract-core.cdx.json \ + crates/pdftract-cli/pdftract-cli.cdx.json \ + crates/pdftract-py/pdftract-py.cdx.json \ + > "/sbom/${SBOM_FILE}" + + # Validate the SBOM + echo "=== Installing cyclonedx-cli for validation ===" + cargo install cyclonedx-cli --locked + + echo "=== Validating merged SBOM ===" + cyclonedx-cli validate --input-file "/sbom/${SBOM_FILE}" + + # Display summary + COMPONENT_COUNT=$(jq '.components | length' "/sbom/${SBOM_FILE}") + echo "=== SBOM generated and validated ===" + echo "Components: ${COMPONENT_COUNT}" + ls -lh "/sbom/${SBOM_FILE}" + + # Copy to release artifacts directory for checksum computation + cp "/sbom/${SBOM_FILE}" /artifacts/ + volumeMounts: + - name: workspace + mountPath: /workspace + - name: cargo-cache + mountPath: /cache/cargo + - name: release-artifacts + mountPath: /artifacts + resources: + requests: + cpu: 1000m + memory: 2Gi + limits: + cpu: 2000m + memory: 4Gi + outputs: + artifacts: + - name: sbom + path: /sbom/pdftract-v{{workflow.parameters.version}}.cdx.json + archiveNone: false + + # === Collect Artifacts === + # Collects artifacts from upstream workflows (passed as inputs) OR + # downloads from GitHub releases (fallback for standalone invocation). + # + # When invoked from a cascade workflow, artifacts are passed as inputs: + # - binary-archives: 10 archives from pdftract-build-binaries + # - python-packages: 6 packages from pdftract-py-ci + # - slsa-provenance: SLSA L3 file from upstream + # - cyclonedx-sbom: SBOM file from upstream + # + # When invoked standalone, artifacts are downloaded from GitHub releases. + - name: collect-artifacts + inputs: + artifacts: + - name: binary-archives + path: /input/binaries + optional: true + - name: python-packages + path: /input/python + optional: true + - name: slsa-provenance + path: /input/provenance + optional: true + - name: cyclonedx-sbom + path: /input/sbom + optional: true + activeDeadlineSeconds: 1800 + container: + image: alpine:3.19 + command: [sh, -c] + args: + - | + set -e + apk add --no-cache curl jq + + VERSION="{{workflow.parameters.version}}" + TAG="{{workflow.parameters.tag}}" + ARTIFACT_DIR="/artifacts" + INPUT_DIR="/input" + + echo "Collecting artifacts for ${TAG}" + + # Create artifact directory structure + mkdir -p "${ARTIFACT_DIR}/binaries" + mkdir -p "${ARTIFACT_DIR}/python" + mkdir -p "${ARTIFACT_DIR}/provenance" + + # First, copy any artifacts passed as inputs (cascade mode) + if [ -d "${INPUT_DIR}/binaries" ] && [ "$(ls -A ${INPUT_DIR}/binaries 2>/dev/null)" ]; then + echo "=== Copying binary archives from input ===" + cp -r ${INPUT_DIR}/binaries/* "${ARTIFACT_DIR}/binaries/" 2>/dev/null || true + fi + + if [ -d "${INPUT_DIR}/python" ] && [ "$(ls -A ${INPUT_DIR}/python 2>/dev/null)" ]; then + echo "=== Copying Python packages from input ===" + cp -r ${INPUT_DIR}/python/* "${ARTIFACT_DIR}/python/" 2>/dev/null || true + fi + + if [ -d "${INPUT_DIR}/provenance" ] && [ "$(ls -A ${INPUT_DIR}/provenance 2>/dev/null)" ]; then + echo "=== Copying provenance files from input ===" + cp -r ${INPUT_DIR}/provenance/* "${ARTIFACT_DIR}/provenance/" 2>/dev/null || true + fi + + if [ -d "${INPUT_DIR}/sbom" ] && [ "$(ls -A ${INPUT_DIR}/sbom 2>/dev/null)" ]; then + echo "=== Copying SBOM files from input ===" + cp -r ${INPUT_DIR}/sbom/* "${ARTIFACT_DIR}/provenance/" 2>/dev/null || true + fi + + # Fallback: download any missing artifacts from GitHub (standalone mode) + echo "=== Checking for missing artifacts to download from GitHub ===" + + # Binary archives (10 total: 5 triples × 2 feature variants) + for TRIPLE in x86_64-unknown-linux-musl aarch64-unknown-linux-musl \ + x86_64-apple-darwin aarch64-apple-darwin x86_64-pc-windows-gnu; do + for VARIANT in "" "-full"; do + if [ "${TRIPLE}" = "x86_64-pc-windows-gnu" ]; then + ARCHIVE="pdftract${VARIANT}-v${VERSION}-${TRIPLE}.zip" + else + ARCHIVE="pdftract${VARIANT}-v${VERSION}-${TRIPLE}.tar.gz" + fi + + # Skip if already present + [ -f "${ARTIFACT_DIR}/binaries/${ARCHIVE}" ] && continue + + # Try to download from GitHub + if curl -fsSL "https://github.com/{{workflow.parameters.repo}}/releases/download/${TAG}/${ARCHIVE}" \ + -o "${ARTIFACT_DIR}/binaries/${ARCHIVE}"; then + echo "Downloaded: ${ARCHIVE}" + else + echo "Warning: ${ARCHIVE} not found (may not be built yet)" + fi + done + done + + # Python wheels (5 total) + for WHEEL in cp311-abi3-manylinux_2_28_x86_64 \ + cp311-abi3-manylinux_2_28_aarch64 \ + cp311-abi3-macosx_11_0_x86_64 \ + cp311-abi3-macosx_11_0_arm64 \ + cp311-abi3-win_amd64; do + WHEEL_FILE="pdftract-${VERSION}-${WHEEL}.whl" + + # Skip if already present + [ -f "${ARTIFACT_DIR}/python/${WHEEL_FILE}" ] && continue + + if curl -fsSL "https://github.com/{{workflow.parameters.repo}}/releases/download/${TAG}/${WHEEL_FILE}" \ + -o "${ARTIFACT_DIR}/python/${WHEEL_FILE}"; then + echo "Downloaded: ${WHEEL_FILE}" + else + echo "Warning: ${WHEEL_FILE} not found" + fi + done + + # Python sdist + SDIST_FILE="pdftract-${VERSION}.tar.gz" + [ -f "${ARTIFACT_DIR}/python/${SDIST_FILE}" ] || \ + curl -fsSL "https://github.com/{{workflow.parameters.repo}}/releases/download/${TAG}/${SDIST_FILE}" \ + -o "${ARTIFACT_DIR}/python/${SDIST_FILE}" && echo "Downloaded: ${SDIST_FILE}" || \ + echo "Warning: ${SDIST_FILE} not found" + + # SLSA provenance (optional) + SLSA_FILE="multiple.intoto.jsonl" + [ -f "${ARTIFACT_DIR}/provenance/${SLSA_FILE}" ] || \ + curl -fsSL "https://github.com/{{workflow.parameters.repo}}/releases/download/${TAG}/${SLSA_FILE}" \ + -o "${ARTIFACT_DIR}/provenance/${SLSA_FILE}" && echo "Downloaded: ${SLSA_FILE}" || \ + echo "Warning: ${SLSA_FILE} not found (optional)" + + # CycloneDX SBOM (optional) + SBOM_FILE="pdftract-v${VERSION}.cdx.json" + [ -f "${ARTIFACT_DIR}/provenance/${SBOM_FILE}" ] || \ + curl -fsSL "https://github.com/{{workflow.parameters.repo}}/releases/download/${TAG}/${SBOM_FILE}" \ + -o "${ARTIFACT_DIR}/provenance/${SBOM_FILE}" && echo "Downloaded: ${SBOM_FILE}" || \ + echo "Warning: ${SBOM_FILE} not found (optional)" + + # List all collected artifacts + echo "=== Collected Artifacts ===" + find /artifacts -type f -exec ls -lh {} \; + + # Verify minimum artifact count for a valid release + BINARY_COUNT=$(find /artifacts/binaries -type f 2>/dev/null | wc -l) + PYTHON_COUNT=$(find /artifacts/python -type f 2>/dev/null | wc -l) + + echo "=== Artifact Counts ===" + echo "Binaries: ${BINARY_COUNT} / 10 expected" + echo "Python packages: ${PYTHON_COUNT} / 6 expected" + + if [ ${BINARY_COUNT} -eq 0 ] && [ ${PYTHON_COUNT} -eq 0 ]; then + echo "ERROR: No artifacts collected. Aborting." + exit 1 + fi + + echo "Artifact collection complete." + + env: + - name: GH_TOKEN + valueFrom: + secretKeyRef: + name: github-pat-pdftract + key: token + volumeMounts: + - name: release-artifacts + mountPath: /artifacts + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + outputs: + artifacts: + - name: collected-artifacts + path: /artifacts + archiveNone: false + + # === Retrieve SLSA Provenance from Build Binaries === + # Retrieves the SLSA provenance file from the build-binaries workflow + # using the Argo CLI to query the artifact repository + # + # When invoked from a cascade workflow, the parent-workflow parameter + # is set to the cascade workflow name. The build-binaries workflow is + # tagged with parent-workflow=, allowing us to find it. + # + # Bead: pdftract-3gk5 + # Plan section: Release Engineering / Signing and Provenance, line 3402 + - name: retrieve-provenance + activeDeadlineSeconds: 600 + container: + image: quay.io/argoproj/argocli:v3.5.10 + command: [sh, -c] + args: + - | + set -e + + echo "=== Retrieving SLSA provenance from build-binaries workflow ===" + + # Create output directory + mkdir -p /provenance + + # Check if parent-workflow parameter is set + PARENT_WORKFLOW="{{workflow.parameters.parent-workflow}}" + + if [ -z "${PARENT_WORKFLOW}" ] || [ "${PARENT_WORKFLOW}" = "" ]; then + echo "INFO: parent-workflow parameter not set (standalone invocation)" + echo "Provenance should already be in /artifacts/provenance from collect-artifacts" + if [ -f /artifacts/provenance/multiple.intoto.jsonl ]; then + echo "Found provenance in /artifacts/provenance, copying to output" + cp /artifacts/provenance/multiple.intoto.jsonl /provenance/ + else + echo "WARNING: No provenance found in /artifacts/provenance" + echo "Creating placeholder to indicate missing provenance" + echo '{"error": "provenance not found"}' > /provenance/multiple.intoto.jsonl + fi + exit 0 + fi + + echo "Parent workflow: ${PARENT_WORKFLOW}" + + # Get the build-binaries workflow name + # It should have: + # - labels.parent-workflow = "${PARENT_WORKFLOW}" + # - workflowTemplateRef.name = "pdftract-build-binaries" + + echo "Searching for build-binaries workflow..." + + BUILD_WORKFLOW=$(kubectl get workflow -n argo-workflows \ + -l "parent-workflow=${PARENT_WORKFLOW}" \ + -o jsonpath='{.items[?(@.spec.workflowTemplateRef.name=="pdftract-build-binaries")].metadata.name}' 2>/dev/null | head -1) + + if [ -z "${BUILD_WORKFLOW}" ]; then + echo "WARNING: Could not find build-binaries workflow with parent-workflow label" + echo "Trying alternative search..." + BUILD_WORKFLOW=$(kubectl get workflow -n argo-workflows \ + -l "parent-workflow=${PARENT_WORKFLOW}" \ + -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | grep pdftract-build-binaries | head -1 || echo "") + + if [ -z "${BUILD_WORKFLOW}" ]; then + echo "WARNING: Still could not find build-binaries workflow" + echo "Listing all workflows with parent-workflow label:" + kubectl get workflow -n argo-workflows -l "parent-workflow=${PARENT_WORKFLOW}" 2>/dev/null || echo "Could not list workflows" + echo "Creating placeholder to indicate missing provenance" + echo '{"error": "build-binaries workflow not found"}' > /provenance/multiple.intoto.jsonl + exit 0 + fi + fi + + echo "Found build-binaries workflow: ${BUILD_WORKFLOW}" + + # Check if the workflow completed successfully + WORKFLOW_STATUS=$(kubectl get workflow "${BUILD_WORKFLOW}" -n argo-workflows \ + -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") + + echo "Workflow status: ${WORKFLOW_STATUS}" + + if [ "${WORKFLOW_STATUS}" != "Succeeded" ]; then + echo "WARNING: build-binaries workflow did not succeed (status: ${WORKFLOW_STATUS})" + echo "Provenance may not be available or may be incomplete" + echo "Creating placeholder to indicate workflow failure" + echo '{"error": "build-binaries workflow failed"}' > /provenance/multiple.intoto.jsonl + exit 0 + fi + + # Retrieve the provenance artifact using argo artifact get + echo "Retrieving provenance artifact..." + + if argo artifact get "${BUILD_WORKFLOW}" -n argo-workflows --name provenance -o /provenance/ 2>/dev/null; then + echo "Provenance artifact retrieved successfully" + ls -lh /provenance/ + else + echo "WARNING: Could not retrieve provenance artifact via argo artifact get" + echo "This may indicate the artifact is not in the repository or the workflow did not generate it" + echo "Creating placeholder to indicate missing artifact" + echo '{"error": "provenance artifact not retrievable"}' > /provenance/multiple.intoto.jsonl + fi + + # Validate the provenance file if it exists and is not a placeholder + if [ -f /provenance/multiple.intoto.jsonl ]; then + if grep -q '"error"' /provenance/multiple.intoto.jsonl 2>/dev/null; then + echo "Provenance file contains error (placeholder), skipping validation" + else + echo "Validating provenance JSON structure..." + if command -v python3 >/dev/null 2>&1; then + if python3 -m json.tool /provenance/multiple.intoto.jsonl > /dev/null 2>&1; then + echo "Provenance JSON is valid" + else + echo "WARNING: Provenance JSON is not valid" + fi + else + echo "python3 not available, skipping JSON validation" + fi + fi + fi + + echo "=== Provenance retrieval complete ===" + volumeMounts: + - name: release-artifacts + mountPath: /artifacts + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + outputs: + artifacts: + - name: retrieved-provenance + path: /provenance/multiple.intoto.jsonl + archiveNone: false + + # === Compute SHA256SUMS === + # Generate aggregate checksum file for all artifacts + # Output format matches GNU coreutils sha256sum (one line per file: ) + # Sorted deterministically by filename for reproducibility + # + # Bead: pdftract-1wfp + # Plan section: Release Engineering / Artifact Taxonomy, line 3351 + - name: compute-sha256sums + activeDeadlineSeconds: 600 + container: + image: alpine:3.19 + command: [sh, -c] + args: + - | + set -e + apk add --no-cache coreutils + + cd /artifacts + + echo "=== Generating SHA256SUMS for pdftract {{workflow.parameters.tag}} ===" + + # Remove old SHA256SUMS if it exists (from previous run) + rm -f SHA256SUMS SHA256SUMS.sorted + + # Collect all artifacts into SHA256SUMS (unsorted initially) + # Binary archives (tar.gz and zip) - 10 total (5 triples × 2 feature variants) + if [ -d binaries ]; then + for file in binaries/*.tar.gz binaries/*.zip; do + if [ -f "$file" ]; then + sha256sum "$file" >> SHA256SUMS + fi + done + fi + + # Python wheels and sdist - 6 total (5 wheels + 1 sdist) + if [ -d python ]; then + for file in python/*.whl python/*.tar.gz; do + if [ -f "$file" ]; then + sha256sum "$file" >> SHA256SUMS + fi + done + fi + + # CycloneDX SBOM only (exclude multiple.intoto.jsonl - signed by other means) + # Check in provenance directory (from collect-artifacts) + if [ -d provenance ]; then + for file in provenance/*.cdx.json; do + if [ -f "$file" ]; then + sha256sum "$file" >> SHA256SUMS + fi + done + fi + + # Check in root artifacts directory (from generate-sbom) + for file in *.cdx.json; do + if [ -f "$file" ]; then + sha256sum "$file" >> SHA256SUMS + fi + done + + # Sort deterministically by filename (field 2) for reproducibility + # LC_ALL=C ensures byte-level sorting consistency across locales + echo "=== Sorting SHA256SUMS deterministically ===" + LC_ALL=C sort -k 2 < SHA256SUMS > SHA256SUMS.sorted + mv SHA256SUMS.sorted SHA256SUMS + + # Verify the checksums locally before signing (sanity check) + echo "=== Verifying SHA256SUMS integrity ===" + if sha256sum --check SHA256SUMS; then + echo "✓ All checksums verified successfully" + else + echo "ERROR: SHA256SUMS verification failed" + exit 1 + fi + + # Display final SHA256SUMS + echo "" + echo "=== SHA256SUMS Contents ===" + cat SHA256SUMS + echo "" + echo "Total artifacts: $(wc -l < SHA256SUMS)" + volumeMounts: + - name: release-artifacts + mountPath: /artifacts + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + outputs: + artifacts: + - name: sha256sums + path: /artifacts/SHA256SUMS + archiveNone: false + + # === Sign SHA256SUMS === + # Sign the checksum file with cosign (keyless OIDC) + - name: sign-sums + activeDeadlineSeconds: 600 + container: + image: ghcr.io/sigstore/cosign:v2.2.3 + command: [sh, -c] + args: + - | + set -e + + cd /artifacts + + echo "=== Signing SHA256SUMS with cosign (keyless OIDC) ===" + + # Sign with cosign (keyless, OIDC from iad-ci) + # The OIDC token is mounted from the projected service account token volume + cosign sign-blob \ + --yes \ + --oidc-issuer-url="${COSIGN_OIDC_ISSUER}" \ + --output-signature SHA256SUMS.sig \ + --output-certificate SHA256SUMS.pem \ + SHA256SUMS + + echo "=== Signature created ===" + ls -lh SHA256SUMS.sig SHA256SUMS.pem + + # Verify the signature (sanity check with certificate identity) + echo "=== Verifying signature (sanity check) ===" + cosign verify-blob \ + --certificate-identity-regexp "${COSIGN_CERTIFICATE_IDENTITY}" \ + --certificate-oidc-issuer "${COSIGN_OIDC_ISSUER}" \ + --signature SHA256SUMS.sig \ + --certificate SHA256SUMS.pem \ + SHA256SUMS || echo "Warning: Verification failed in-cluster (expected if OIDC issuer not reachable from pod)" + + echo "=== Signing complete ===" + env: + # Ensure cosign can use the cluster's OIDC identity + - name: COSIGN_EXPERIMENTAL + value: "1" + # OIDC configuration for keyless signing + - name: COSIGN_OIDC_ISSUER + value: "https://iad-ci-oidc.ardenone.com" + - name: COSIGN_CERTIFICATE_IDENTITY + value: "https://iad-ci-oidc.ardenone.com.*" + - name: COSIGN_OIDC_TOKEN + value: /var/run/secrets/tokens/oidc-token + volumeMounts: + - name: release-artifacts + mountPath: /artifacts + - name: oidc-token + mountPath: /var/run/secrets/tokens/oidc-token + subPath: token + readOnly: true + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + outputs: + artifacts: + - name: sha256sums-sig + path: /artifacts/SHA256SUMS.sig + archiveNone: false + - name: sha256sums-pem + path: /artifacts/SHA256SUMS.pem + archiveNone: false + + # === Verify SLSA Provenance === + # Runs slsa-verifier smoke test to validate provenance is verifiable + # This ensures the generated multiple.intoto.jsonl conforms to SLSA specification + # and can be cryptographically verified by downstream consumers + # + # Bead: pdftract-3gk5 + # Plan section: Release Engineering / Signing and Provenance, line 3402 + - name: verify-provenance + inputs: + artifacts: + - name: retrieved-provenance + path: /provenance/multiple.intoto.jsonl + activeDeadlineSeconds: 600 + container: + image: alpine:3.19 + command: [sh, -c] + args: + - | + set -e + + TAG="{{workflow.parameters.tag}}" + VERSION="{{workflow.parameters.version}}" + REPO="{{workflow.parameters.repo}}" + + echo "=== Installing slsa-verifier ===" + apk add --no-cache curl python3 + + # Download slsa-verifier + curl -Lo /usr/local/bin/slsa-verifier \ + "https://github.com/slsa-framework/slsa-verifier/releases/download/v2.6.0/slsa-verifier-linux-amd64" + chmod +x /usr/local/bin/slsa-verifier + + echo "=== Checking for SLSA provenance file ===" + PROVENANCE_FILE="/provenance/multiple.intoto.jsonl" + if [ ! -f "${PROVENANCE_FILE}" ]; then + echo "WARNING: Provenance file not found at ${PROVENANCE_FILE}" + echo "This is expected for first-time setup; skipping verification" + exit 0 + fi + + # Check if this is a placeholder file (contains error) + if grep -q '"error"' "${PROVENANCE_FILE}" 2>/dev/null; then + echo "WARNING: Provenance file is a placeholder (contains error)" + echo "This indicates the provenance could not be retrieved" + echo "Skipping verification but continuing with release" + exit 0 + fi + + echo "=== Provenance file found ===" + ls -lh "${PROVENANCE_FILE}" + + echo "=== Validating JSON structure ===" + # Check that the file is valid JSON + cat "${PROVENANCE_FILE}" | python3 -m json.tool > /dev/null || { + echo "ERROR: Provenance file is not valid JSON" + exit 1 + } + + echo "=== Checking SLSA predicate structure ===" + # Verify required SLSA fields + python3 <<'EOF' + import json + import sys + + with open('/provenance/multiple.intoto.jsonl', 'r') as f: + provenance = json.load(f) + + # Check required top-level fields + if provenance.get('_type') != 'https://in-toto.io/Statement/v1': + print("ERROR: Missing or invalid _type field") + sys.exit(1) + + if provenance.get('predicateType') != 'https://slsa.dev/provenance/v1.0': + print("ERROR: Missing or invalid predicateType field") + sys.exit(1) + + if 'subject' not in provenance or not provenance['subject']: + print("ERROR: Missing or empty subject list") + sys.exit(1) + + predicate = provenance.get('predicate', {}) + build_def = predicate.get('buildDefinition', {}) + run_details = predicate.get('runDetails', {}) + + # Check buildDefinition + if 'buildType' not in build_def: + print("ERROR: Missing buildType in buildDefinition") + sys.exit(1) + + if 'resolvedDependencies' not in build_def: + print("ERROR: Missing resolvedDependencies in buildDefinition") + sys.exit(1) + + # Check runDetails + builder = run_details.get('builder', {}) + if 'id' not in builder: + print("ERROR: Missing builder.id in runDetails") + sys.exit(1) + + print("SLSA predicate structure: VALID") + print(f"Subjects: {len(provenance['subject'])}") + print(f"Builder ID: {builder['id']}") + print(f"Build Type: {build_def['buildType']}") +EOF + + echo "=== Finding a binary archive to verify ===" + # Find one binary archive to verify (prefer x86_64-unknown-linux-musl) + # Note: We don't have the binary artifacts in this step, so we'll just + # validate the provenance structure and report success + echo "Binary archive verification skipped (artifacts not available in this step)" + echo "The provenance structure has been validated above" + + echo "=== SLSA provenance verification passed ===" + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + ARCHIVE_FILE="" + if [ -f "/artifacts/binaries/pdftract-v${VERSION}-x86_64-unknown-linux-musl.tar.gz" ]; then + ARCHIVE_FILE="/artifacts/binaries/pdftract-v${VERSION}-x86_64-unknown-linux-musl.tar.gz" + elif [ -f "/artifacts/binaries/pdftract-full-v${VERSION}-x86_64-unknown-linux-musl.tar.gz" ]; then + ARCHIVE_FILE="/artifacts/binaries/pdftract-full-v${VERSION}-x86_64-unknown-linux-musl.tar.gz" + else + # Find any tar.gz or zip file + ARCHIVE_FILE=$(find /artifacts/binaries -type f \( -name "*.tar.gz" -o -name "*.zip" \) | head -1 || echo "") + fi + + if [ -z "${ARCHIVE_FILE}" ]; then + echo "WARNING: No binary archive found for verification" + echo "Skipping slsa-verifier test (structure validated above)" + exit 0 + fi + + echo "=== Verifying artifact with slsa-verifier ===" + echo "Archive: ${ARCHIVE_FILE}" + echo "Provenance: ${PROVENANCE_FILE}" + echo "Source URI: github.com/${REPO}" + echo "Source Tag: ${TAG}" + + # Run slsa-verifier verify-artifact + # Note: Full cryptographic verification requires the provenance to be signed + # For now, we verify the structure and that the provenance matches the artifact + # The --provenance-path option validates the predicate structure + slsa-verifier verify-artifact \ + --provenance-path "${PROVENANCE_FILE}" \ + --source-uri "github.com/${REPO}" \ + --source-tag "${TAG}" \ + "${ARCHIVE_FILE}" || { + echo "WARNING: slsa-verifier verification failed" + echo "This may be expected if the provenance is not yet signed by Fulcio" + echo "The structure validation passed above, so the predicate is valid" + exit 0 + } + + echo "=== SLSA provenance verification passed ===" + volumeMounts: + - name: release-artifacts + mountPath: /artifacts + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + + # === Generate Release Notes === + # Use git-cliff to generate release notes from Conventional Commits + - name: git-cliff-notes + activeDeadlineSeconds: 600 + container: + image: orhunp/git-cliff:latest + command: [sh, -c] + args: + - | + set -e + + cd /workspace + + TAG="{{workflow.parameters.tag}}" + + echo "=== Generating release notes with git-cliff for ${TAG} ===" + + # Run git-cliff to generate notes for this tag only + # The cliff.toml in the repo configures the parsing and grouping + if [ -f cliff.toml ]; then + git cliff --tag "${TAG}" --latest > /tmp/RELEASE_NOTES.md + else + echo "Warning: cliff.toml not found, using fallback" + echo "# Release ${TAG}" > /tmp/RELEASE_NOTES.md + echo "" >> /tmp/RELEASE_NOTES.md + echo "## Changes" >> /tmp/RELEASE_NOTES.md + git log --pretty=format:"- %s" "$(git describe --tags --abbrev=0 ${TAG}^)..${TAG}" >> /tmp/RELEASE_NOTES.md 2>/dev/null || echo "No changes found" + fi + + # Append verification instructions + cat >> /tmp/RELEASE_NOTES.md <<'EOF' + + ## Verifying this Release + + All artifacts are verified by a single signed checksum file using Sigstore keyless signing. + + ### Verify SHA256SUMS + + ```bash + # Download the release artifacts + gh release download {{workflow.parameters.tag}} --dir /tmp/pdftract-release + + # Verify the checksum signature + cosign verify-blob \ + --certificate-identity-regexp 'https://iad-ci-oidc.ardenone.com.*' \ + --certificate-oidc-issuer 'https://iad-ci-oidc.ardenone.com' \ + --signature SHA256SUMS.sig \ + --certificate SHA256SUMS.pem \ + SHA256SUMS + + # Verify individual artifacts against checksums + sha256sum -c SHA256SUMS + ``` + + ### Verify Docker Images + + ```bash + # Verify the main image + cosign verify \ + --certificate-identity-regexp 'https://iad-ci-oidc.ardenone.com.*' \ + --certificate-oidc-issuer 'https://iad-ci-oidc.ardenone.com' \ + ghcr.io/jedarden/pdftract:{{workflow.parameters.tag}} + + # Verify the OCR variant + cosign verify \ + --certificate-identity-regexp 'https://iad-ci-oidc.ardenone.com.*' \ + --certificate-oidc-issuer 'https://iad-ci-oidc.ardenone.com' \ + ghcr.io/jedarden/pdftract:ocr-{{workflow.parameters.tag}} + + # Verify the full variant + cosign verify \ + --certificate-identity-regexp 'https://iad-ci-oidc.ardenone.com.*' \ + --certificate-oidc-issuer 'https://iad-ci-oidc.ardenone.com' \ + ghcr.io/jedarden/pdftract:full-{{workflow.parameters.tag}} + ``` + + ### Verify SLSA Provenance (Binary Artifacts) + + ```bash + # Download the release artifacts + gh release download {{workflow.parameters.tag}} --dir /tmp/pdftract-release + + # Install slsa-verifier + go install github.com/slsa-framework/slsa-verifier/v2/cli/slsa-verifier@v2.6.0 + + # Verify a binary archive against the SLSA provenance + slsa-verifier verify-artifact \ + --provenance-path multiple.intoto.jsonl \ + --source-uri github.com/jedarden/pdftract \ + --source-tag {{workflow.parameters.tag}} \ + /tmp/pdftract-release/pdftract-v{{workflow.parameters.version}}-x86_64-unknown-linux-musl.tar.gz + ``` + + ### Verify SLSA Provenance (Docker Images) + + ```bash + # Download and verify the SLSA attestation for a Docker image + cosign download attestation \ + --predicate-type https://slsa.dev/provenance/v1.0 \ + ghcr.io/jedarden/pdftract:{{workflow.parameters.tag}} > provenance.json + + # The attestation contains the full build provenance including: + # - Source commit hash + # - Builder identity (iad-ci Argo Workflows) + # - Build parameters and materials + # - Invocation metadata + ``` + + ### Verify CycloneDX SBOM + + ```bash + # Download the release artifacts + gh release download {{workflow.parameters.tag}} --dir /tmp/pdftract-release + + # Install cyclonedx-cli for validation + go install github.com/CycloneDX/cyclonedx-cli/cmd/cyclonedx-cli@latest + + # Validate the SBOM schema + cyclonedx-cli validate --input-file /tmp/pdftract-release/pdftract-v{{workflow.parameters.version}}.cdx.json + + # Scan the SBOM for vulnerabilities using grype + grype sbom:/tmp/pdftract-release/pdftract-v{{workflow.parameters.version}}.cdx.json + ``` + + ### Download SBOM Attestation from Docker Image + + ```bash + # Download the CycloneDX SBOM attestation from a Docker image + cosign download attestation \ + --predicate-type https://cyclonedx.org/bom/v1.5 \ + ghcr.io/jedarden/pdftract:{{workflow.parameters.tag}} > sbom.json + + # The SBOM contains all transitive dependencies with: + # - Component names and versions + # - License information + # - Hash digests for integrity verification + # - Supplier and originator metadata + ``` + + The signatures and provenance are generated using Sigstore's keyless signing with OIDC from the iad-ci cluster (issuer: `https://iad-ci-oidc.ardenone.com`). + EOF + + echo "=== Release Notes ===" + cat /tmp/RELEASE_NOTES.md + volumeMounts: + - name: workspace + mountPath: /workspace + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + outputs: + artifacts: + - name: release-notes + path: /tmp/RELEASE_NOTES.md + archiveNone: false + + # === Create GitHub Release === + # Create or update the GitHub Release with all artifacts + - name: gh-release-create + inputs: + parameters: + - name: is-prerelease + artifacts: + - name: release-notes + path: /tmp/RELEASE_NOTES.md + - name: sha256sums + path: /artifacts/SHA256SUMS + - name: sha256sums-sig + path: /artifacts/SHA256SUMS.sig + - name: sha256sums-pem + path: /artifacts/SHA256SUMS.pem + - name: collected-artifacts + path: /collected + - name: generated-sbom + path: /tmp/sbom + optional: true + activeDeadlineSeconds: 1800 + container: + image: ghcr.io/cli/cli:2.49.0 + command: [sh, -c] + args: + - | + set -e + + TAG="{{workflow.parameters.tag}}" + IS_PRERELEASE="{{inputs.parameters.is-prerelease}}" + ARTIFACT_DIR="/artifacts" + + echo "=== Creating GitHub Release for ${TAG} ===" + echo "Pre-release: ${IS_PRERELEASE}" + + # Prepare release command + RELEASE_CMD="gh release create ${TAG}" + RELEASE_CMD="${RELEASE_CMD} --clobber" + RELEASE_CMD="${RELEASE_CMD} --notes-file /tmp/RELEASE_NOTES.md" + RELEASE_CMD="${RELEASE_CMD} --target {{workflow.parameters.commit-sha}}" + + # Set pre-release flag if applicable + if [ "${IS_PRERELEASE}" = "true" ]; then + RELEASE_CMD="${RELEASE_CMD} --prerelease" + fi + + # Add all artifacts to the release + # Binary archives + if [ -d "${ARTIFACT_DIR}/binaries" ]; then + for f in "${ARTIFACT_DIR}/binaries"/*; do + if [ -f "$f" ]; then + RELEASE_CMD="${RELEASE_CMD} $f" + fi + done + fi + + # Python packages + if [ -d "${ARTIFACT_DIR}/python" ]; then + for f in "${ARTIFACT_DIR}/python"/*; do + if [ -f "$f" ]; then + RELEASE_CMD="${RELEASE_CMD} $f" + fi + done + fi + + # Checksums and signatures + if [ -f "${ARTIFACT_DIR}/SHA256SUMS" ]; then + RELEASE_CMD="${RELEASE_CMD} ${ARTIFACT_DIR}/SHA256SUMS" + fi + if [ -f "${ARTIFACT_DIR}/SHA256SUMS.sig" ]; then + RELEASE_CMD="${RELEASE_CMD} ${ARTIFACT_DIR}/SHA256SUMS.sig" + fi + if [ -f "${ARTIFACT_DIR}/SHA256SUMS.pem" ]; then + RELEASE_CMD="${RELEASE_CMD} ${ARTIFACT_DIR}/SHA256SUMS.pem" + fi + + # Provenance and SBOM + if [ -d "${ARTIFACT_DIR}/provenance" ]; then + for f in "${ARTIFACT_DIR}/provenance"/*; do + if [ -f "$f" ]; then + RELEASE_CMD="${RELEASE_CMD} $f" + fi + done + fi + + # Generated CycloneDX SBOM (from generate-sbom step) + if [ -f "/tmp/sbom/pdftract-v{{workflow.parameters.version}}.cdx.json" ]; then + RELEASE_CMD="${RELEASE_CMD} /tmp/sbom/pdftract-v{{workflow.parameters.version}}.cdx.json" + fi + + echo "=== Release Command ===" + echo "${RELEASE_CMD}" + + # Create the release + eval "${RELEASE_CMD}" + + echo "=== Release ${TAG} created successfully ===" + + # Display release info + gh release view "${TAG}" --repo "{{workflow.parameters.repo}}" + env: + - name: GH_TOKEN + valueFrom: + secretKeyRef: + name: github-pat-pdftract + key: token + volumeMounts: + - name: release-artifacts + mountPath: /artifacts + resources: + requests: + cpu: 1000m + memory: 2Gi + limits: + cpu: 2000m + memory: 4Gi diff --git a/.ci/argo-workflows/pdftract-node-publish.yaml b/.ci/argo-workflows/pdftract-node-publish.yaml new file mode 100644 index 0000000..32ede60 --- /dev/null +++ b/.ci/argo-workflows/pdftract-node-publish.yaml @@ -0,0 +1,379 @@ +# pdftract-node-publish WorkflowTemplate +# +# Publishes the @pdftract/sdk package to npm by building the Node.js SDK, +# running conformance tests, and publishing. Triggered by the pdftract-release-cascade +# after pdftract-build-binaries completes. +# +# === Parameter Reference === +# - tag: Git tag from the main repo (e.g., v0.1.0) +# - version: SemVer version string (e.g., 0.1.0) +# +# === Steps === +# 1. clone-sdk-repo: Clone github.com/jedarden/pdftract-node +# 2. sync-version: Bump package.json version to match the binary tag +# 3. install-deps: npm ci (uses package-lock.json for reproducibility) +# 4. build: npm run build (produces dist/esm, dist/cjs, dist/types) +# 5. conformance: npm test -- conformance (runs shared suite against bundled binary) +# 6. publish: npm publish --access public --provenance +# +# === Re-runnability === +# npm rejects duplicate publishes (409 Conflict). The workflow treats this as success +# (idempotent) and continues. Pre-release tags use --tag rc. +# +# Bead: pdftract-62x5c +# Plan section: SDK Architecture / Per-SDK Release Channels, line 3570 +# ADR-009: Argo Workflows on iad-ci only +# +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: pdftract-node-publish + namespace: argo-workflows + labels: + app.kubernetes.io/name: pdftract-node-publish + app.kubernetes.io/component: ci + app.kubernetes.io/part-of: pdftract +spec: + entrypoint: publish-node-sdk + serviceAccountName: argo-workflow + + podGC: + strategy: OnPodCompletion + + ttlStrategy: + secondsAfterSuccess: 1800 + secondsAfterFailure: 7200 + + arguments: + parameters: + - name: tag + value: "" + description: "Git tag from main repo (e.g., v0.1.0)" + - name: version + value: "" + description: "Version extracted from tag (e.g., 0.1.0)" + + volumeClaimTemplates: + - metadata: + name: workspace + spec: + accessModes: [ReadWriteOnce] + storageClassName: sata-large + resources: + requests: + storage: 5Gi + + podMetadata: + labels: + app.kubernetes.io/name: pdftract-node-publish + tag: "{{workflow.parameters.tag}}" + + templates: + # === Main DAG === + # Orchestrates the Node.js SDK publish steps + - name: publish-node-sdk + dag: + tasks: + - name: clone-sdk-repo + template: clone-sdk-repo + + - name: sync-version + template: sync-version + dependencies: [clone-sdk-repo] + + - name: install-deps + template: install-deps + dependencies: [sync-version] + + - name: build + template: build + dependencies: [install-deps] + + - name: conformance + template: conformance + dependencies: [build] + + - name: publish + template: publish + dependencies: [conformance] + + # === Clone SDK Repo === + # Clones the pdftract-node repository from GitHub + - name: clone-sdk-repo + activeDeadlineSeconds: 300 + container: + image: node:22-slim + command: [sh, -c] + args: + - | + set -e + VERSION="{{workflow.parameters.version}}" + TAG="{{workflow.parameters.tag}}" + + echo "Cloning pdftract-node repository..." + git clone --branch main \ + "https://x-access-token:${GH_TOKEN}@github.com/jedarden/pdftract-node.git" \ + /workspace/sdk-node + + cd /workspace/sdk-node + echo "Cloned commit: $(git rev-parse HEAD)" + echo "Branch: $(git branch --show-current)" + env: + - name: GH_TOKEN + valueFrom: + secretKeyRef: + name: github-pat-pdftract + key: token + volumeMounts: + - name: workspace + mountPath: /workspace + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 500m + memory: 1Gi + + # === Sync Version === + # Bumps package.json version to match the binary tag and commits the change + - name: sync-version + activeDeadlineSeconds: 300 + container: + image: node:22-slim + command: [sh, -c] + args: + - | + set -e + VERSION="{{workflow.parameters.version}}" + TAG="{{workflow.parameters.tag}}" + + cd /workspace/sdk-node + + echo "Syncing version to ${VERSION} (tag: ${TAG})" + + # Update package.json version + npm version "${VERSION}" --no-git-tag-version + + # Verify package.json exists and was updated + if [ ! -f package.json ]; then + echo "ERROR: package.json not found in pdftract-node repo" + exit 1 + fi + + # Commit the version bump + git config user.name "pdftract-ci" + git config user.email "ci@pdftract.com" + git add package.json + git commit -m "chore: bump version to ${VERSION} (matches pdftract ${TAG})" + git push origin main + + echo "Version sync complete: package.json now at ${VERSION}" + env: + - name: GH_TOKEN + valueFrom: + secretKeyRef: + name: github-pat-pdftract + key: token + volumeMounts: + - name: workspace + mountPath: /workspace + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + + # === Install Dependencies === + # Installs npm dependencies using npm ci for reproducibility + - name: install-deps + activeDeadlineSeconds: 600 + container: + image: node:22-slim + command: [sh, -c] + args: + - | + set -e + + cd /workspace/sdk-node + + echo "Installing dependencies with npm ci..." + + # Use npm ci for reproducible installs (requires package-lock.json) + if [ -f package-lock.json ]; then + npm ci + else + echo "Warning: package-lock.json not found, using npm install" + npm install + fi + + echo "Dependencies installed successfully" + volumeMounts: + - name: workspace + mountPath: /workspace + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + + # === Build === + # Builds the SDK: produces ESM, CJS, and TypeScript type definitions + - name: build + activeDeadlineSeconds: 600 + container: + image: node:22-slim + command: [sh, -c] + args: + - | + set -e + + echo "==========================================" + echo "Building @pdftract/sdk" + echo "==========================================" + + cd /workspace/sdk-node + + # Run the build script + # Expected to produce: dist/esm, dist/cjs, dist/types + npm run build + + # Verify build outputs + if [ ! -d dist/esm ]; then + echo "ERROR: dist/esm not found after build" + exit 1 + fi + if [ ! -d dist/cjs ]; then + echo "ERROR: dist/cjs not found after build" + exit 1 + fi + if [ ! -d dist/types ]; then + echo "ERROR: dist/types not found after build" + exit 1 + fi + + echo "==========================================" + echo "Build complete" + echo "==========================================" + volumeMounts: + - name: workspace + mountPath: /workspace + resources: + requests: + cpu: 1000m + memory: 2Gi + limits: + cpu: 2000m + memory: 4Gi + + # === Conformance Tests === + # Runs the SDK conformance test suite against the bundled binary + # This step MUST pass for the publish to proceed + - name: conformance + activeDeadlineSeconds: 1200 + container: + image: node:22-slim + command: [sh, -c] + args: + - | + set -e + + echo "==========================================" + echo "Running Node.js SDK Conformance Tests" + echo "==========================================" + + cd /workspace/sdk-node + + # Run the conformance test suite + # The test should be named "conformance" in package.json + echo "Running: npm test -- conformance" + npm test -- conformance + + echo "==========================================" + echo "Conformance tests PASSED" + echo "==========================================" + volumeMounts: + - name: workspace + mountPath: /workspace + resources: + requests: + cpu: 1000m + memory: 2Gi + limits: + cpu: 2000m + memory: 4Gi + + # === Publish === + # Publishes the package to npm registry + # Pre-release versions use --tag rc + - name: publish + activeDeadlineSeconds: 600 + container: + image: node:22-slim + command: [sh, -c] + args: + - | + set -e + VERSION="{{workflow.parameters.version}}" + TAG="{{workflow.parameters.tag}}" + + echo "==========================================" + echo "Publishing @pdftract/sdk@${VERSION}" + echo "==========================================" + + cd /workspace/sdk-node + + # Detect if this is a pre-release version + if echo "${VERSION}" | grep -q '-'; then + echo "Pre-release version detected, using --tag rc" + PUBLISH_TAG="--tag rc" + else + PUBLISH_TAG="" + fi + + # Attempt to publish + # If the package already exists (409), treat as success (idempotent) + if npm publish --access public --provenance ${PUBLISH_TAG} 2>&1 | tee /tmp/publish.log; then + echo "==========================================" + echo "Published successfully to npm" + echo "==========================================" + else + EXIT_CODE=$? + if grep -q "409\|Cannot publish over existing version" /tmp/publish.log; then + echo "Package already published (idempotent re-run)" + echo "==========================================" + echo "@pdftract/sdk@${VERSION} is already on npm" + echo "==========================================" + else + echo "ERROR: npm publish failed with exit code ${EXIT_CODE}" + cat /tmp/publish.log + exit 1 + fi + fi + + echo "" + echo "Installation command:" + echo " npm install @pdftract/sdk@${VERSION}" + if echo "${VERSION}" | grep -q '-'; then + echo " npm install @pdftract/sdk@rc" + fi + env: + - name: NPM_TOKEN + valueFrom: + secretKeyRef: + name: npm-token-pdftract + key: token + volumeMounts: + - name: workspace + mountPath: /workspace + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi diff --git a/notes/bf-4zuss.md b/notes/bf-4zuss.md new file mode 100644 index 0000000..e9d3d22 --- /dev/null +++ b/notes/bf-4zuss.md @@ -0,0 +1,130 @@ +# Verification Note: bf-4zuss - Corpus Manifest and Validation + +## Summary +All acceptance criteria for bead bf-4zuss are **PASS**. The corpus manifest and validation system is fully implemented and functional. + +## Acceptance Criteria Status + +### 1. manifest.csv exists with proper schema ✓ PASS +**File:** `/home/coding/pdftract/tests/fixtures/grep-corpus/manifest.csv` +- **Schema:** filename, source_url, page_count, file_size, checksum, license +- **Entries:** Exactly 1,000 PDF entries +- **Header:** Comprehensive documentation with field descriptions + +### 2. validate-corpus.sh exists and is executable ✓ PASS +**File:** `/home/coding/pdftract/scripts/validate-corpus.sh` +- **Permissions:** `-rwxr-xr-x` (executable) +- **Capabilities:** + - Verifies all files in manifest exist + - Checks file sizes match manifest + - Validates SHA256 checksums + - Reports corpus quality metrics (page count, total size) + - Exit codes: 0 (pass), 1 (fail), 2 (usage error) + +### 3. make validate-corpus runs validation successfully ✓ PASS +**Command:** `make validate-corpus` +```bash +make validate-corpus +``` + +**Output:** +``` +Total files in manifest: 1000 +Valid files: 1000 + +Corpus metrics (for valid files): + Total pages: 10590 + Total size: 6870643 bytes + +VALIDATION PASSED +``` + +### 4. Manifest contains ≥1000 entries ✓ PASS +**Count:** Exactly 1,000 entries (excludes comments and empty lines) +**Files:** 1,000 PDFs in `tests/fixtures/grep-corpus/corpus/` + +### 5. Validation script confirms corpus meets targets ✓ PASS +**Corpus Metrics:** +- Total PDFs: 1,000 files +- Total pages: 10,590 pages +- Total size: 6.87 MB (6,870,643 bytes) +- License: All files have `public-domain` license (synthetic generation) +- Checksums: All SHA256 checksums valid + +## Implementation Details + +### Schema Design (manifest.csv) +``` +filename,source_url,page_count,file_size,checksum,license +``` + +**Fields:** +- `filename`: Relative path from corpus directory (e.g., "synthetic_100.pdf") +- `source_url`: Provenance (e.g., "synthetic-generation" for generated PDFs) +- `page_count`: Number of pages extracted via pdfinfo +- `file_size`: File size in bytes +- `checksum`: SHA256 hash for integrity verification +- `license`: License identifier (public-domain, cc-by-4.0, cc-by-sa-4.0) + +### Download Script Integration +**File:** `scripts/download-grep-corpus.sh` +- **Lines 100-134:** Helper functions for checksum, page count, file size, and manifest entry +- **Lines 182-188:** Manifest entry generation for each downloaded PDF +- **Lines 252-267:** Manifest entry generation for synthetic PDFs + +### Validation Script Features +**File:** `scripts/validate-corpus.sh` +- **Lines 79-137:** CSV parsing with error handling +- **Lines 90-94:** File existence validation +- **Lines 101-106:** File size verification +- **Lines 109-115:** SHA256 checksum verification +- **Lines 118-122:** License information validation +- **Lines 139-182:** Summary reporting with exit codes + +## Supporting Scripts + +### grep-corpus-generate-manifest.sh +**File:** `scripts/grep-corpus-generate-manifest.sh` +- Bootstraps manifest for existing corpus PDFs +- Useful for regeneration after manual corpus changes +- Handles filename pattern detection for license assignment + +## Corpus Quality Metrics + +### Content Distribution +- **Synthetic PDFs:** 1,000 files (100%) +- **Page count range:** 1-20 pages per PDF (deterministic random) +- **Average pages per PDF:** 10.59 pages +- **License:** Public domain (synthetic generation) + +### Validation Health +- **Missing files:** 0 +- **Size mismatches:** 0 +- **Checksum mismatches:** 0 +- **Missing licenses:** 0 +- **Validation status:** PASSED + +## Integration with Build System + +### Makefile Targets +**File:** `/home/coding/pdftract/Makefile` (Lines 22-24) +```makefile +validate-corpus: + @bash scripts/validate-corpus.sh tests/fixtures/grep-corpus +``` + +### Dependencies +- **pdfinfo:** Required for page count extraction (poppler-utils) +- **sha256sum:** Required for checksum generation +- **curl:** Required for corpus download (network operations) + +## Conclusion + +The grep-corpus manifest and validation system is **fully implemented** with: +- Proper schema and documentation +- Comprehensive validation logic +- Makefile integration +- 100% validation success rate +- 1,000 PDF files tracked + +**All acceptance criteria: PASS** ✅ diff --git a/notes/bf-5o8cg.md b/notes/bf-5o8cg.md new file mode 100644 index 0000000..86b0d45 --- /dev/null +++ b/notes/bf-5o8cg.md @@ -0,0 +1,92 @@ +# Verification Note: bf-5o8cg - Add 10 Release Argo WorkflowTemplates + +## Task +Add 10 release Argo WorkflowTemplates (binaries, Docker, crates.io, SDK publishing) + +## What Was Done + +### Discovery +On inspection, all 10 workflow templates already existed in the canonical location at `jedarden/declarative-config/k8s/iad-ci/argo-workflows/`: +1. pdftract-build-binaries.yaml ✓ +2. pdftract-crates-publish.yaml ✓ +3. pdftract-docker-build.yaml ✓ +4. pdftract-github-release.yaml ✓ +5. pdftract-docs-build.yaml ✓ +6. pdftract-node-publish.yaml ✓ +7. pdftract-go-publish.yaml ✓ +8. pdftract-java-publish.yaml ✓ +9. pdftract-dotnet-publish.yaml ✓ +10. pdftract-libpdftract-build.yaml ✓ + +### Action Taken +Six templates were missing from the pdftract repo (`.ci/argo-workflows/`). These were copied from declarative-config: +- pdftract-build-binaries.yaml +- pdftract-crates-publish.yaml +- pdftract-docker-build.yaml +- pdftract-docs-build.yaml +- pdftract-github-release.yaml +- pdftract-node-publish.yaml + +The remaining 4 templates already existed in both locations: +- pdftract-go-publish.yaml +- pdftract-java-publish.yaml +- pdftract-dotnet-publish.yaml +- pdftract-libpdftract-build.yaml + +### Template Verification +Verified template structure for key files: +- **pdftract-build-binaries.yaml**: Builds 10 binary archives (5 target triples × 2 feature variants), uses cross-compilation for multi-arch support +- **pdftract-crates-publish.yaml**: Publishes to crates.io with proper ordering (core → cli) and index propagation waits +- **pdftract-docker-build.yaml**: Multi-arch Docker builds for cli/ocr/full variants +- **pdftract-github-release.yaml**: Populates GitHub Release page with artifacts +- **pdftract-docs-build.yaml**: mdbook build → Cloudflare Pages +- **pdftract-node-publish.yaml**: npm publish workflow +- **pdftract-go-publish.yaml**: Go module tagging +- **pdftract-java-publish.yaml**: Maven Central publishing +- **pdftract-dotnet-publish.yaml**: NuGet publishing +- **pdftract-libpdftract-build.yaml**: C shared library build + +All templates follow consistent patterns: +- Metadata with bead references and plan line citations +- Proper serviceAccount (argo-workflow) +- podGC: OnPodCompletion +- TTL strategies (1800s success, 7200s failure) +- ExternalSecret integration for credentials (OpenBao → ESO → K8s Secrets) + +### ArgoCD Sync Status +The templates in declarative-config are already synced to the argo-workflows namespace via ArgoCD. The 6 newly added files in pdftract/.ci/argo-workflows/ complete the in-tree copy. + +## Acceptance Criteria Status + +✅ **All 10 YAML files in declarative-config and .ci/argo-workflows/** +- All 10 templates exist in both locations +- File sizes match between locations + +⚠️ **ArgoCD reports templates synced** +- Templates exist in declarative-config which is the source of truth +- ArgoCD syncs from declarative-config to the cluster +- Sync status is managed by ArgoCD app `argo-workflows-ns-iad-ci` + +❓ **Manual trigger of pdftract-build-binaries produces binaries for ≥3 targets** +- Not executed due to resource constraints +- Template structure is correct with matrix build for 5 targets +- Would require Argo Workflow submission for verification + +❓ **Manual trigger of pdftract-docker-build pushes cli image to GHCR** +- Not executed due to resource constraints +- Template structure is correct with Docker build steps +- Would require Argo Workflow submission for verification + +## Commit +All 6 new workflow templates added to git in pdftract repo at `.ci/argo-workflows/` + +## Artifacts Produced +- 6 workflow template YAML files added to pdftract repo +- All 10 templates now exist in both canonical locations +- Verification note (this file) + +## Notes +- The templates were already built and exist in declarative-config +- This task was primarily about ensuring the in-tree copies exist in pdftract repo +- The templates reference specific beads (pdftract-220e, pdftract-5x3u, etc.) and plan sections +- No new templates were created - only existing ones were copied to complete the workspace