Repository navigation
e2e Test - refactor-smoke C1 #9139
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End-to-End Tests | |
| run-name: e2e Test - ${{ inputs.test-name || inputs.generate-cli-command || github.event.inputs.generate-cli-command }} | |
| permissions: | |
| contents: read | |
| # Independent GPU dispatches queue through the priority scheduler; newer runs must not cancel them. | |
| on: # zizmor: ignore[concurrency-limits] | |
| workflow_dispatch: | |
| inputs: | |
| generate-cli-command: | |
| description: "Command passed to generate matrix script" | |
| required: false | |
| type: string | |
| default: "" | |
| klaud-run: | |
| description: "Run Klaud Cold benchmarks at background priority" | |
| required: false | |
| type: boolean | |
| default: false | |
| test-name: | |
| description: "Name for this test run" | |
| required: false | |
| type: string | |
| ref: | |
| description: "Ref (branch/sha) to checkout for generating configs" | |
| required: false | |
| type: string | |
| duration-override: | |
| description: "Override matrix.config.duration (seconds). Empty = use matrix value." | |
| required: false | |
| type: string | |
| default: "" | |
| require-power: | |
| description: "Fail multi-node benchmark jobs when GPU power is invalid" | |
| required: false | |
| type: boolean | |
| default: false | |
| power-producer-sha: | |
| description: "Expected srt-slurm producer git commit for power validation" | |
| required: false | |
| type: string | |
| default: "" | |
| agentx-fast: | |
| description: "AgentX fast feedback: one warmup request per lane and a 20-minute profile." | |
| required: false | |
| type: boolean | |
| default: false | |
| eval-limit: | |
| description: "Eval instance count: empty = whole split (default); N = first-N smoke slice" | |
| required: false | |
| type: string | |
| default: "" | |
| eval-framework: | |
| description: "Eval runner override (auto uses model-aware selection; lm-eval, kimi-vendor, minimax-vendor, or bfcl)" | |
| required: false | |
| type: string | |
| default: "auto" | |
| eval-suite: | |
| description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval" | |
| required: false | |
| type: string | |
| default: "" | |
| changelog-base-ref: | |
| description: "Trusted-dispatch mode: base SHA for infx.matrix.plan" | |
| required: false | |
| type: string | |
| default: "" | |
| changelog-head-ref: | |
| description: "Trusted-dispatch mode: exact approved PR head SHA" | |
| required: false | |
| type: string | |
| default: "" | |
| trim-conc: | |
| description: "Trim each configuration to its minimum concurrency" | |
| required: false | |
| type: boolean | |
| default: false | |
| all-evals: | |
| description: "Expand eval selection to every fixed-sequence config; eval-only with generate-cli-command, changelog mode keeps throughput unless evals-only" | |
| required: false | |
| type: boolean | |
| default: false | |
| evals-only: | |
| description: "Suppress throughput and run only evaluations" | |
| required: false | |
| type: boolean | |
| default: false | |
| fail-fast: | |
| description: "Cancel the rest of each matrix after its first failure" | |
| required: false | |
| type: boolean | |
| default: false | |
| workflow_call: | |
| secrets: | |
| INFERENCEX_OFFICIAL_RO_HF_TOKEN: | |
| required: true | |
| inputs: | |
| generate-cli-command: | |
| description: "Command passed to generate matrix script" | |
| required: false | |
| type: string | |
| default: "" | |
| klaud-run: | |
| description: "Run Klaud Cold benchmarks at background priority" | |
| required: false | |
| type: boolean | |
| default: false | |
| test-name: | |
| description: "Name for this test run" | |
| required: false | |
| type: string | |
| ref: | |
| description: "Ref (branch/sha) to checkout for generating configs" | |
| required: false | |
| type: string | |
| duration-override: | |
| description: "Override matrix.config.duration (seconds). Empty = use matrix value." | |
| required: false | |
| type: string | |
| default: "" | |
| require-power: | |
| description: "Fail multi-node benchmark jobs when GPU power is invalid" | |
| required: false | |
| type: boolean | |
| default: false | |
| power-producer-sha: | |
| description: "Expected srt-slurm producer git commit for power validation" | |
| required: false | |
| type: string | |
| default: "" | |
| agentx-fast: | |
| description: "AgentX fast feedback: one warmup request per lane and a 20-minute profile." | |
| required: false | |
| type: boolean | |
| default: false | |
| eval-limit: | |
| description: "Eval instance count: empty = whole split (default); N = first-N smoke slice" | |
| required: false | |
| type: string | |
| default: "" | |
| eval-framework: | |
| description: "Eval runner override (auto uses model-aware selection; lm-eval, kimi-vendor, minimax-vendor, or bfcl)" | |
| required: false | |
| type: string | |
| default: "auto" | |
| eval-suite: | |
| description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval" | |
| required: false | |
| type: string | |
| default: "" | |
| changelog-base-ref: | |
| description: "Trusted-dispatch mode: base SHA for infx.matrix.plan" | |
| required: false | |
| type: string | |
| default: "" | |
| changelog-head-ref: | |
| description: "Trusted-dispatch mode: exact approved PR head SHA" | |
| required: false | |
| type: string | |
| default: "" | |
| trim-conc: | |
| description: "Trim each configuration to its minimum concurrency" | |
| required: false | |
| type: boolean | |
| default: false | |
| all-evals: | |
| description: "Expand eval selection to every fixed-sequence config; eval-only with generate-cli-command, changelog mode keeps throughput unless evals-only" | |
| required: false | |
| type: boolean | |
| default: false | |
| evals-only: | |
| description: "Suppress throughput and run only evaluations" | |
| required: false | |
| type: boolean | |
| default: false | |
| fail-fast: | |
| description: "Cancel the rest of each matrix after its first failure" | |
| required: false | |
| type: boolean | |
| default: false | |
| env: | |
| PYTHONPATH: ${{ github.workspace }}/inferencex-e2e | |
| jobs: | |
| get-jobs: | |
| name: get-jobs | |
| runs-on: ubuntu-latest | |
| outputs: | |
| benchmark-ref: ${{ steps.checkout-ref.outputs.commit || steps.checkout-default.outputs.commit }} | |
| tooling-ref: ${{ steps.tooling.outputs.result }} | |
| single-node-config: ${{ steps.get-jobs.outputs.single-node-config }} | |
| multi-node-config: ${{ steps.get-jobs.outputs.multi-node-config }} | |
| eval-config: ${{ steps.get-jobs.outputs.eval-config }} | |
| multi-node-eval-config: ${{ steps.get-jobs.outputs.multi-node-eval-config }} | |
| agentic-config: ${{ steps.get-jobs.outputs.agentic-config }} | |
| agentic-eval-config: ${{ steps.get-jobs.outputs.agentic-eval-config }} | |
| multi-node-agentic-config: ${{ steps.get-jobs.outputs.multi-node-agentic-config }} | |
| multi-node-agentic-eval-config: ${{ steps.get-jobs.outputs.multi-node-agentic-eval-config }} | |
| steps: | |
| - name: Checkout code (ref) | |
| id: checkout-ref | |
| if: ${{ inputs.ref && inputs.ref != '' }} | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ inputs.ref }} | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - name: Checkout code (default) | |
| id: checkout-default | |
| if: ${{ !inputs.ref || inputs.ref == '' }} | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.sha }} | |
| persist-credentials: false | |
| - name: Resolve workflow tooling | |
| id: tooling | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| env: | |
| CHECKOUT_SHA: ${{ steps.checkout-ref.outputs.commit || steps.checkout-default.outputs.commit }} | |
| with: | |
| result-encoding: string | |
| script: | | |
| const { data } = await github.rest.repos.getCommit({ | |
| ...context.repo, ref: context.payload.repository.default_branch, | |
| }); | |
| await core.summary.addHeading('Revisions', 2).addTable([ | |
| ['Setup checkout', process.env.CHECKOUT_SHA], | |
| ['infx tooling', data.sha], | |
| ]).write(); | |
| return data.sha; | |
| - name: Checkout workflow tooling | |
| if: ${{ inputs.ref && inputs.ref != '' }} | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.workflow_sha }} | |
| path: .ci-priority | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 | |
| - id: get-jobs | |
| env: | |
| GENERATE_COMMAND: ${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }} | |
| CHANGELOG_BASE_REF: ${{ inputs.changelog-base-ref }} | |
| CHANGELOG_HEAD_REF: ${{ inputs.changelog-head-ref }} | |
| TRIM_CONC: ${{ inputs.trim-conc }} | |
| ALL_EVALS: ${{ inputs.all-evals }} | |
| EVALS_ONLY: ${{ inputs.evals-only }} | |
| run: | | |
| if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi | |
| MEASURED_ROOT="$PWD" | |
| PRIORITY_ROOT="${GITHUB_WORKSPACE}" | |
| if [ -d "${GITHUB_WORKSPACE}/.ci-priority" ]; then | |
| PRIORITY_ROOT="${GITHUB_WORKSPACE}/.ci-priority" | |
| fi | |
| if [[ -f "$PRIORITY_ROOT/inferencex-e2e/configs/runners.yaml" ]]; then | |
| PRIORITY_ROOT="$PRIORITY_ROOT/inferencex-e2e" | |
| fi | |
| env PYTHONPATH="$PRIORITY_ROOT" python3 -P -m infx.workflows.require_launcher "$GITHUB_WORKSPACE" | |
| if [ -n "$CHANGELOG_BASE_REF" ] || [ -n "$CHANGELOG_HEAD_REF" ]; then | |
| if [ -z "$CHANGELOG_BASE_REF" ] || [ -z "$CHANGELOG_HEAD_REF" ]; then | |
| echo "Both changelog-base-ref and changelog-head-ref are required" >&2 | |
| exit 1 | |
| fi | |
| CMD=( | |
| env PYTHONPATH="$PRIORITY_ROOT" | |
| uv run --no-project --exclude-newer PT12H --python 3.12 --with pydantic --with pyyaml | |
| python -P -m infx.matrix.revision plan "$MEASURED_ROOT" | |
| --changelog-file "${MEASURED_ROOT}/perf-changelog.yaml" | |
| --base-ref "$CHANGELOG_BASE_REF" | |
| --head-ref "$CHANGELOG_HEAD_REF" | |
| ) | |
| if [ "$TRIM_CONC" = "true" ]; then | |
| CMD+=(--trim-conc) | |
| fi | |
| if [ "$ALL_EVALS" = "true" ]; then | |
| CMD+=(--all-evals) | |
| fi | |
| if [ "$EVALS_ONLY" = "true" ]; then | |
| CMD+=(--evals-only) | |
| fi | |
| RAW_CONFIG_JSON=$("${CMD[@]}") | |
| CONFIG_JSON=$(python3 -c 'import json,sys; data=json.load(sys.stdin); rows=[row for family in ("single_node","multi_node") for group in data.get(family,{}).values() for row in group]; rows.extend(row for family in ("evals","agentic_evals","multinode_evals","multinode_agentic_evals") for row in data.get(family,[])); print(json.dumps(rows))' <<<"$RAW_CONFIG_JSON") | |
| else | |
| if [ -z "$GENERATE_COMMAND" ]; then | |
| echo "generate-cli-command is required outside trusted changelog dispatch mode" >&2 | |
| exit 1 | |
| fi | |
| read -r -a GENERATE_ARGS <<< "$GENERATE_COMMAND" | |
| if [ "$TRIM_CONC" = "true" ]; then | |
| GENERATE_ARGS+=(--trim-conc) | |
| fi | |
| if [ "$ALL_EVALS" = "true" ]; then | |
| GENERATE_ARGS+=(--all-evals) | |
| fi | |
| if [ "$EVALS_ONLY" = "true" ]; then | |
| GENERATE_ARGS+=(--evals-only) | |
| fi | |
| CONFIG_JSON=$(env PYTHONPATH="$PRIORITY_ROOT" \ | |
| uv run --no-project --exclude-newer PT12H --python 3.12 --with pydantic --with pyyaml \ | |
| python -P -m infx.matrix.revision generate "$MEASURED_ROOT" "${GENERATE_ARGS[@]}") | |
| fi | |
| CONFIG_JSON=$(printf '%s' "$CONFIG_JSON" | env PYTHONPATH="$PRIORITY_ROOT" \ | |
| uv run --no-project --exclude-newer PT12H --python 3.12 --with pydantic --with pyyaml \ | |
| python -P -m infx.workflows.benchmark_schema) | |
| score_matrix() { | |
| local family="$1" | |
| env PYTHONPATH="$PRIORITY_ROOT" uv run --no-project --exclude-newer PT12H --python 3.12 --with pyyaml \ | |
| python -P -m infx.workflows.ci_priority \ | |
| --policy "${PRIORITY_ROOT}/configs/ci-priority.yaml" \ | |
| --event-name "${GITHUB_EVENT_NAME}" \ | |
| --queue-namespace "${GITHUB_RUN_ID}:${GITHUB_RUN_ATTEMPT}:${family}" | |
| } | |
| AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and not x.get('eval-only', False)]))" | score_matrix agentic) | |
| AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and x.get('run-eval', False)]))" | score_matrix agentic-eval) | |
| MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and not x.get('eval-only', False)]))" | score_matrix multi-agentic) | |
| MULTI_AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-agentic-eval) | |
| SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single) | |
| MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi) | |
| EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix eval) | |
| MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix multi-eval) | |
| { | |
| echo "agentic-config=$AGENTIC" | |
| echo "agentic-eval-config=$AGENTIC_EVAL" | |
| echo "multi-node-agentic-config=$MULTI_AGENTIC" | |
| echo "multi-node-agentic-eval-config=$MULTI_AGENTIC_EVAL" | |
| echo "single-node-config=$SINGLE" | |
| echo "multi-node-config=$MULTI" | |
| echo "eval-config=$EVALS" | |
| echo "multi-node-eval-config=$MULTI_EVAL" | |
| } >> "$GITHUB_OUTPUT" | |
| test-sweep-multi-node: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.multi-node-config != '' && needs.get-jobs.outputs.multi-node-config != '[]' }} | |
| uses: $/.github/workflows/benchmark-multinode-tmpl.yml | |
| name: multi-node / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.multi-node-config) }} | |
| secrets: | |
| INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} | |
| with: | |
| config: ${{ toJSON(matrix.config) }} | |
| klaud-run: ${{ inputs.klaud-run }} | |
| runner: ${{ matrix.config.runner }} | |
| node-count: ${{ matrix.config.node-count }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| conc-list: ${{ toJson(matrix.config.conc) }} | |
| run-eval: false | |
| require-power: ${{ inputs.require-power || (matrix.config['require-power'] == true && matrix.config['eval-only'] != true) }} | |
| power-producer-sha: ${{ inputs.power-producer-sha }} | |
| ref: ${{ needs.get-jobs.outputs.benchmark-ref }} | |
| test-sweep-multi-node-evals: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.multi-node-eval-config != '' && needs.get-jobs.outputs.multi-node-eval-config != '[]' }} | |
| uses: $/.github/workflows/benchmark-multinode-tmpl.yml | |
| name: multi-node eval / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.multi-node-eval-config) }} | |
| secrets: | |
| INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} | |
| with: | |
| config: ${{ toJSON(matrix.config) }} | |
| klaud-run: ${{ inputs.klaud-run }} | |
| runner: ${{ matrix.config.runner }} | |
| node-count: ${{ matrix.config.node-count }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| conc-list: ${{ toJson(matrix.config.conc) }} | |
| run-eval: true | |
| eval-only: true | |
| eval-conc: ${{ (inputs.eval-framework == 'auto' && (matrix.config['eval-framework'] || 'lm-eval') || inputs.eval-framework) == 'lm-eval' && matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }} | |
| eval-limit: ${{ inputs.eval-limit }} | |
| eval-framework: ${{ inputs.eval-framework == 'auto' && (matrix.config['eval-framework'] || 'lm-eval') || inputs.eval-framework }} | |
| eval-suite: ${{ inputs.eval-suite != '' && inputs.eval-suite || (inputs.eval-framework == 'auto' && matrix.config['eval-suite'] || '') }} | |
| ref: ${{ needs.get-jobs.outputs.benchmark-ref }} | |
| test-sweep-agentic: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.agentic-config != '' && needs.get-jobs.outputs.agentic-config != '[]' }} | |
| uses: $/.github/workflows/benchmark-tmpl.yml | |
| name: agentic / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.agentic-config) }} | |
| secrets: | |
| INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} | |
| with: | |
| config: ${{ toJSON(matrix.config) }} | |
| klaud-run: ${{ inputs.klaud-run }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| duration: ${{ inputs.agentx-fast && '1200' || (inputs.duration-override != '' && inputs.duration-override || matrix.config.duration) }} | |
| agentx-fast: ${{ inputs.agentx-fast }} | |
| run-eval: false | |
| scenario-type: agentic-coding | |
| ref: ${{ needs.get-jobs.outputs.benchmark-ref }} | |
| test-sweep-agentic-evals: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.agentic-eval-config != '' && needs.get-jobs.outputs.agentic-eval-config != '[]' }} | |
| uses: $/.github/workflows/benchmark-tmpl.yml | |
| name: agentic eval / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.agentic-eval-config) }} | |
| secrets: | |
| INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} | |
| with: | |
| config: ${{ toJSON(matrix.config) }} | |
| klaud-run: ${{ inputs.klaud-run }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| duration: ${{ inputs.agentx-fast && '1200' || (inputs.duration-override != '' && inputs.duration-override || matrix.config.duration) }} | |
| agentx-fast: ${{ inputs.agentx-fast }} | |
| run-eval: true | |
| eval-only: true | |
| eval-limit: ${{ inputs.eval-limit }} | |
| eval-framework: ${{ inputs.eval-framework == 'auto' && (matrix.config['eval-framework'] || 'lm-eval') || inputs.eval-framework }} | |
| eval-suite: ${{ inputs.eval-suite != '' && inputs.eval-suite || (inputs.eval-framework == 'auto' && matrix.config['eval-suite'] || '') }} | |
| scenario-type: agentic-coding | |
| ref: ${{ needs.get-jobs.outputs.benchmark-ref }} | |
| test-sweep-multi-node-agentic: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.multi-node-agentic-config != '' && needs.get-jobs.outputs.multi-node-agentic-config != '[]' }} | |
| uses: $/.github/workflows/benchmark-multinode-tmpl.yml | |
| name: multi-node agentic / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.multi-node-agentic-config) }} | |
| secrets: | |
| INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} | |
| with: | |
| config: ${{ toJSON(matrix.config) }} | |
| klaud-run: ${{ inputs.klaud-run }} | |
| runner: ${{ matrix.config.runner }} | |
| node-count: ${{ matrix.config.node-count }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| conc-list: ${{ toJson(matrix.config.conc) }} | |
| conc: ${{ matrix.config.conc[0] }} | |
| duration: ${{ inputs.agentx-fast && '1200' || (inputs.duration-override != '' && inputs.duration-override || matrix.config.duration) }} | |
| agentx-fast: ${{ inputs.agentx-fast }} | |
| run-eval: false | |
| require-power: ${{ inputs.require-power }} | |
| power-producer-sha: ${{ inputs.power-producer-sha }} | |
| scenario-type: agentic-coding | |
| ref: ${{ needs.get-jobs.outputs.benchmark-ref }} | |
| test-sweep-multi-node-agentic-evals: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.multi-node-agentic-eval-config != '' && needs.get-jobs.outputs.multi-node-agentic-eval-config != '[]' }} | |
| uses: $/.github/workflows/benchmark-multinode-tmpl.yml | |
| name: multi-node agentic eval / | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.multi-node-agentic-eval-config) }} | |
| secrets: | |
| INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} | |
| with: | |
| config: ${{ toJSON(matrix.config) }} | |
| klaud-run: ${{ inputs.klaud-run }} | |
| runner: ${{ matrix.config.runner }} | |
| node-count: ${{ matrix.config.node-count }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| conc-list: ${{ format('[{0}]', matrix.config['eval-conc']) }} | |
| conc: ${{ matrix.config['eval-conc'] }} | |
| duration: ${{ inputs.duration-override != '' && inputs.duration-override || matrix.config.duration }} | |
| run-eval: true | |
| eval-only: true | |
| eval-conc: ${{ matrix.config['eval-conc'] }} | |
| eval-limit: ${{ inputs.eval-limit }} | |
| eval-framework: ${{ inputs.eval-framework == 'auto' && (matrix.config['eval-framework'] || 'lm-eval') || inputs.eval-framework }} | |
| eval-suite: ${{ inputs.eval-suite != '' && inputs.eval-suite || (inputs.eval-framework == 'auto' && matrix.config['eval-suite'] || '') }} | |
| scenario-type: agentic-coding | |
| ref: ${{ needs.get-jobs.outputs.benchmark-ref }} | |
| test-sweep-single-node: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.single-node-config != '' && needs.get-jobs.outputs.single-node-config != '[]' }} | |
| uses: $/.github/workflows/benchmark-tmpl.yml | |
| name: single-node / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.single-node-config) }} | |
| secrets: | |
| INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} | |
| with: | |
| config: ${{ toJSON(matrix.config) }} | |
| klaud-run: ${{ inputs.klaud-run }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| run-eval: false | |
| ref: ${{ needs.get-jobs.outputs.benchmark-ref }} | |
| test-sweep-evals: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.eval-config != '' && needs.get-jobs.outputs.eval-config != '[]' }} | |
| uses: $/.github/workflows/benchmark-tmpl.yml | |
| name: eval / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.eval-config) }} | |
| secrets: | |
| INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} | |
| with: | |
| config: ${{ toJSON(matrix.config) }} | |
| klaud-run: ${{ inputs.klaud-run }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| run-eval: true | |
| eval-only: true | |
| eval-limit: ${{ inputs.eval-limit }} | |
| eval-framework: ${{ inputs.eval-framework == 'auto' && (matrix.config['eval-framework'] || 'lm-eval') || inputs.eval-framework }} | |
| eval-suite: ${{ inputs.eval-suite != '' && inputs.eval-suite || (inputs.eval-framework == 'auto' && matrix.config['eval-suite'] || '') }} | |
| ref: ${{ needs.get-jobs.outputs.benchmark-ref }} | |
| collect-results: | |
| name: collect-results | |
| needs: [get-jobs, test-sweep-multi-node, test-sweep-single-node, test-sweep-agentic, test-sweep-multi-node-agentic] | |
| if: ${{ always() && (needs.test-sweep-multi-node.result != 'skipped' || needs.test-sweep-single-node.result != 'skipped' || needs.test-sweep-agentic.result != 'skipped' || needs.test-sweep-multi-node-agentic.result != 'skipped') }} | |
| uses: $/.github/workflows/collect-results.yml | |
| with: | |
| result-prefix: "bmk" | |
| tooling-ref: ${{ needs.get-jobs.outputs.tooling-ref }} | |
| collect-evals: | |
| name: collect-evals | |
| needs: [get-jobs, test-sweep-evals, test-sweep-multi-node-evals, test-sweep-agentic-evals, test-sweep-multi-node-agentic-evals] | |
| if: ${{ always() && (needs.test-sweep-evals.result != 'skipped' || needs.test-sweep-multi-node-evals.result != 'skipped' || needs.test-sweep-agentic-evals.result != 'skipped' || needs.test-sweep-multi-node-agentic-evals.result != 'skipped') }} | |
| uses: $/.github/workflows/collect-evals.yml | |
| with: | |
| tooling-ref: ${{ needs.get-jobs.outputs.tooling-ref }} | |
| collect-job-events: | |
| name: collect-job-events | |
| needs: [get-jobs, test-sweep-multi-node, test-sweep-multi-node-evals, test-sweep-agentic, test-sweep-agentic-evals, test-sweep-multi-node-agentic, test-sweep-multi-node-agentic-evals, test-sweep-single-node, test-sweep-evals] | |
| if: ${{ !cancelled() && needs.get-jobs.result == 'success' }} | |
| uses: $/.github/workflows/collect-job-events.yml | |
| with: | |
| tooling-ref: ${{ needs.get-jobs.outputs.tooling-ref }} | |
| calc-success-rate: | |
| name: calc-success-rate | |
| needs: [get-jobs, collect-results, collect-evals] | |
| if: ${{ always() && needs.get-jobs.outputs.tooling-ref != '' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: read # Read job timings for run statistics. | |
| contents: read | |
| env: | |
| RESULTS_DIR: "results/" | |
| STATS_FILENAME: "run_stats" | |
| GITHUB_TOKEN: ${{ github.token }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ needs.get-jobs.outputs.tooling-ref }} | |
| fetch-depth: 1 | |
| persist-credentials: false | |
| - name: Download results artifacts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| path: ${{ env.RESULTS_DIR }} | |
| pattern: results_* | |
| - name: Select Python project | |
| run: | | |
| python_project="$GITHUB_WORKSPACE" | |
| if [[ -f "$python_project/inferencex-e2e/pyproject.toml" ]]; then | |
| python_project="$python_project/inferencex-e2e" | |
| fi | |
| test -f "$python_project/pyproject.toml" | |
| echo "INFERENCEX_PYTHON_PROJECT=$python_project" >> "$GITHUB_ENV" | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 | |
| with: | |
| cache-dependency-glob: | | |
| pyproject.toml | |
| uv.lock | |
| inferencex-e2e/pyproject.toml | |
| inferencex-e2e/uv.lock | |
| - name: Calculate success rate | |
| run: >- | |
| uv run --project "$INFERENCEX_PYTHON_PROJECT" --locked --extra workflows | |
| python -m infx.workflows.calc_success_rate "$STATS_FILENAME" | |
| # Exits non-zero after writing valid rows when it quarantines any. | |
| - if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: "run-stats" | |
| path: ${{ env.STATS_FILENAME }}.json | |
| - if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: rejected_rows_run_stats | |
| path: rejected_rows.json | |
| if-no-files-found: ignore |