action.yml

name: agent-trace eval
description: Run agent-trace evals in CI, post results to the step summary, and fail on regression.
author: Siddhant-K-code
branding:
  icon: activity
  color: purple

inputs:
  config:
    description: Path to eval config file
    default: .agent-evals.yaml
  baseline:
    description: Path to baseline scores file (optional)
    default: ""
  save-baseline:
    description: Path to save current scores as a new baseline (optional)
    default: ""
  tolerance:
    description: Regression tolerance (0.0–1.0, default 0.05)
    default: "0.05"
  trace-dir:
    description: Directory where agent-trace sessions are stored
    default: .agent-traces
  python-version:
    description: Python version to use
    default: "3.12"
  install-extras:
    description: Comma-separated optional extras to install (e.g. openai,anthropic)
    default: ""
  post-pr-comment:
    description: Post or update an agent session summary on the pull request
    default: "false"
  github-token:
    description: GitHub token used only when post-pr-comment is true
    default: ""

outputs:
  passed:
    description: "true if all scorers passed, false otherwise"
    value: ${{ steps.eval.outputs.passed }}
  summary-path:
    description: Path to the written eval summary markdown file
    value: ${{ steps.eval.outputs.summary-path }}

runs:
  using: composite
  steps:
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ inputs.python-version }}

    - name: Install agent-trace
      shell: bash
      run: |
        EXTRAS="${{ inputs.install-extras }}"
        if [ -n "$EXTRAS" ]; then
          pip install "agent-strace[$EXTRAS]"
        else
          pip install agent-strace
        fi

    - name: Run evals
      id: eval
      shell: bash
      env:
        EVAL_CONFIG: ${{ inputs.config }}
        EVAL_BASELINE: ${{ inputs.baseline }}
        EVAL_SAVE_BASELINE: ${{ inputs.save-baseline }}
        EVAL_TOLERANCE: ${{ inputs.tolerance }}
        EVAL_TRACE_DIR: ${{ inputs.trace-dir }}
      run: |
        ARGS=(--config "$EVAL_CONFIG" --tolerance "$EVAL_TOLERANCE")

        if [ -n "$EVAL_BASELINE" ]; then
          ARGS+=(--baseline "$EVAL_BASELINE")
        fi

        if [ -n "$EVAL_SAVE_BASELINE" ]; then
          ARGS+=(--save-baseline "$EVAL_SAVE_BASELINE")
        fi

        if agent-strace --trace-dir "$EVAL_TRACE_DIR" eval "${ARGS[@]}"; then
          echo "passed=true" >> "$GITHUB_OUTPUT"
        else
          echo "passed=false" >> "$GITHUB_OUTPUT"
          exit 1
        fi

        if [ -n "$GITHUB_STEP_SUMMARY" ] && [ -f "$GITHUB_STEP_SUMMARY" ]; then
          echo "summary-path=$GITHUB_STEP_SUMMARY" >> "$GITHUB_OUTPUT"
        fi

    - name: Upload trace artifacts
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: agent-traces
        path: ${{ inputs.trace-dir }}
        if-no-files-found: ignore

    - name: Post agent-trace PR comment
      if: ${{ always() && inputs.post-pr-comment == 'true' }}
      shell: bash
      env:
        GITHUB_TOKEN: ${{ inputs.github-token }}
        PR_COMMENT_TRACE_DIR: ${{ inputs.trace-dir }}
        PR_HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }}
        PR_BASE_REPOSITORY: ${{ github.repository }}
      run: |
        if [ -n "$PR_HEAD_REPOSITORY" ] && [ "$PR_HEAD_REPOSITORY" != "$PR_BASE_REPOSITORY" ]; then
          echo "::notice::Skipping PR comment for a fork because pull_request tokens are read-only."
        elif find "$PR_COMMENT_TRACE_DIR" -mindepth 2 -name meta.json -print -quit 2>/dev/null | grep -q .; then
          agent-strace --trace-dir "$PR_COMMENT_TRACE_DIR" pr-comment
        else
          echo "::notice::No agent-trace sessions found; skipping PR comment."
        fi