name: Prime Agent benchmarks
run-name: 'Prime Agent benchmarks · PR #${{ github.event.pull_request.number || inputs.pr }}'
on:
pull_request_target:
types: [opened, synchronize, reopened, ready_for_review]
workflow_dispatch:
inputs:
pr:
description: Open PR number to benchmark against current main
required: true
type: number
concurrency:
group: prime-agent-benchmark-${{ github.event.pull_request.number || inputs.pr }}
cancel-in-progress: true
permissions: {}
jobs:
request:
if: github.event_name == 'pull_request_target' || github.ref == 'refs/heads/main'
runs-on: ubuntu-24.04
timeout-minutes: 5
concurrency:
group: 'prime-agent-benchmark-comment-Prime Agent benchmarks · PR #${{ github.event.pull_request.number || inputs.pr }}'
cancel-in-progress: false
permissions:
contents: read
actions: read
pull-requests: write
outputs:
allowed: ${{ steps.resolve.outputs.needed == 'true' && steps.vouch.outputs.vouched == 'true' }}
harness: ${{ steps.resolve.outputs.harness }}
steps:
# pull_request_target supplies the default-branch SHA, including for stacked PRs.
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ github.sha }}
persist-credentials: false
- name: Install benchmark tooling
run: |
python3 -m venv .benchmark-bootstrap
.benchmark-bootstrap/bin/pip install uv==0.12.9
.benchmark-bootstrap/bin/uv sync --locked --project scripts/benchmarks
echo "$PWD/.benchmark-bootstrap/bin" >> "$GITHUB_PATH"
- name: Resolve exact revisions
id: resolve
working-directory: scripts/benchmarks
run: uv run --locked cli.py resolve --results request
env:
GITHUB_TOKEN: ${{ github.token }}
- name: Check contributor trust
id: vouch
if: steps.resolve.outputs.needed == 'true'
uses: mitchellh/vouch/action/check-user@d66fa29a64600490892131ad87597c30c91fcac4 # v1
with:
user: ${{ steps.resolve.outputs.author }}
allow-fail: true
env:
GITHUB_TOKEN: ${{ github.token }}
- name: Update benchmark comment
if: steps.resolve.outputs.needed == 'true'
working-directory: scripts/benchmarks
run: |
if [ "$VOUCHED" = true ]; then
uv run --locked cli.py publish --results request
else
uv run --locked cli.py publish --results request --pending-trust
fi
env:
GITHUB_TOKEN: ${{ github.token }}
VOUCHED: ${{ steps.vouch.outputs.vouched }}
- name: Save benchmark request
if: steps.resolve.outputs.needed == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0
with:
name: benchmark-request-${{ github.run_attempt }}
path: scripts/benchmarks/request/report.json
retention-days: 14
benchmark:
needs: request
if: needs.request.outputs.allowed == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 25
permissions:
contents: read
actions: read
pull-requests: read
steps:
# The controller is trusted main code. PR code executes only in Prime sandboxes.
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ needs.request.outputs.harness }}
persist-credentials: false
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.0
with:
name: benchmark-request-${{ github.run_attempt }}
path: scripts/benchmarks/request
- name: Install benchmark tooling
run: |
python3 -m venv .benchmark-bootstrap
.benchmark-bootstrap/bin/pip install uv==0.12.9
.benchmark-bootstrap/bin/uv sync --locked --project scripts/benchmarks
echo "$PWD/.benchmark-bootstrap/bin" >> "$GITHUB_PATH"
- name: Benchmark main and PR in Prime sandboxes
working-directory: scripts/benchmarks
run: uv run --locked cli.py run
env:
GITHUB_TOKEN: ${{ github.token }}
PRIME_SANDBOX_API_KEY: ${{ secrets.PRIME_SANDBOX_API_KEY }}
PRIME_TEAM_ID: ${{ vars.PRIME_BENCHMARK_TEAM_ID }}
- name: Save benchmark results
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0
with:
name: benchmark-results-${{ github.run_attempt }}
path: scripts/benchmarks/results/
if-no-files-found: warn
retention-days: 14