Skip to content
CICD NeMo RL

feat(csd): add cross-tokenizer distillation and sub-token preparation #17460

Sign in to view logs

feat(csd): add cross-tokenizer distillation and sub-token preparation

feat(csd): add cross-tokenizer distillation and sub-token preparation #17460

Workflow file for this run

.github/workflows/cicd-main.yml at 22ebcf7
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: "CICD NeMo RL"
on:
push:
branches:
- main
- "pull-request/[0-9]+"
schedule:
- cron: "0 9 * * *"
workflow_dispatch:
inputs:
test_to_run:
required: false
default: L2
type: choice
options:
- docs
- Lfast
- L0
- L1
- L2
description: Test level to run. docs = doc tests only (reuses main container), Lfast = fast subset (reuses main container), L0 = unit/docs/lint, L1 = L0 + functional, L2 = L1 + convergence
test_script_pattern:
description: "Case-insensitive substring used to select unit and functional test script names (for example, Megatron_1 or Vllm)."
required: false
default: ""
type: string
test_platform:
description: "Select which test job matrices run. Unit and Lfast tests are H100-only."
required: false
default: all
type: choice
options:
- all
- h100
- gb200
megatron_lm_commit:
description: "Full Megatron-LM commit SHA to check out in the Megatron-Bridge submodule."
required: false
default: ""
type: string
image_tag:
description: "Override container image tag (e.g. 'main'). Skips container build."
required: false
default: ""
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name || 'main' }}-${{ github.event_name }}
cancel-in-progress: true
env:
GB200_CONTAINER_REGISTRY: ${{ vars.GB200_CONTAINER_REGISTRY }}
jobs:
pre-flight:
runs-on: ubuntu-latest
outputs:
test_level: ${{ steps.evaluate.outputs.test_level }}
image_tag: ${{ steps.evaluate.outputs.image_tag }}
test_script_pattern: ${{ steps.evaluate.outputs.test_script_pattern }}
test_platform: ${{ steps.evaluate.outputs.test_platform }}
unit_test_scripts: ${{ steps.evaluate.outputs.unit_test_scripts }}
unit_test_count: ${{ steps.evaluate.outputs.unit_test_count }}
h100_functional_test_scripts: ${{ steps.evaluate.outputs.h100_functional_test_scripts }}
h100_functional_test_count: ${{ steps.evaluate.outputs.h100_functional_test_count }}
gb200_functional_test_scripts: ${{ steps.evaluate.outputs.gb200_functional_test_scripts }}
gb200_functional_test_count: ${{ steps.evaluate.outputs.gb200_functional_test_count }}
megatron_lm_commit: ${{ steps.evaluate.outputs.megatron_lm_commit }}
base_ref: ${{ steps.base-head-ref.outputs.base_ref }}
base_sha: ${{ steps.base-head-ref.outputs.base_sha }}
head_ref: ${{ steps.base-head-ref.outputs.head_ref }}
head_sha: ${{ steps.base-head-ref.outputs.head_sha }}
head_label: ${{ steps.base-head-ref.outputs.head_label }}
has_skip_cicd: ${{ steps.base-head-ref.outputs.has_skip_cicd }}
test_sha: ${{ steps.base-head-ref.outputs.test_sha }}
steps:
- name: Get PR info
id: get-pr-info
if: startsWith(github.ref, 'refs/heads/pull-request/')
uses: nv-gha-runners/get-pr-info@090577647b8ddc4e06e809e264f7881650ecdccf # main
- name: Checkout
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
with:
fetch-depth: 0
- name: Determine base and head references
id: base-head-ref
env:
IS_PULL_REQUEST_REF: ${{ startsWith(github.ref, 'refs/heads/pull-request/') }}
PR_INFO_JSON: ${{ steps.get-pr-info.outputs.pr-info }}
run: |
if [[ "$IS_PULL_REQUEST_REF" == "true" && -n "$PR_INFO_JSON" ]]; then
base_ref=$(echo "$PR_INFO_JSON" | jq -r '.base.ref')
base_sha=$(echo "$PR_INFO_JSON" | jq -r '.base.sha')
head_ref=$(echo "$PR_INFO_JSON" | jq -r '.head.ref')
head_sha=$(echo "$PR_INFO_JSON" | jq -r '.head.sha')
test_sha=$(echo "$PR_INFO_JSON" | jq -r '.merge_commit_sha')
head_label=$(echo "$PR_INFO_JSON" | jq -r '.head.label // empty')
ci_label=$(echo "$PR_INFO_JSON" | jq -r '[.labels[]? | (if type == "string" then . else .name end) | select(startswith("CI:"))] | first // empty')
has_skip_cicd=$(echo "$PR_INFO_JSON" | jq -r '[.labels[]? | (if type == "string" then . else .name end) | select(. == "Skip CICD")] | length > 0')
else
base_ref="HEAD~1"
base_sha=$(git rev-parse HEAD~1)
head_ref="HEAD"
head_sha="${{ github.sha }}"
test_sha="${{ github.sha }}"
head_label="${{ github.ref_name }}"
ci_label=""
has_skip_cicd="false"
fi
[[ "$has_skip_cicd" != "true" ]] && has_skip_cicd="false"
echo "base_ref=$base_ref" >> "$GITHUB_OUTPUT"
echo "base_sha=$base_sha" >> "$GITHUB_OUTPUT"
echo "head_ref=$head_ref" >> "$GITHUB_OUTPUT"
echo "head_sha=$head_sha" >> "$GITHUB_OUTPUT"
echo "test_sha=$test_sha" >> "$GITHUB_OUTPUT"
echo "head_label=$head_label" >> "$GITHUB_OUTPUT"
echo "ci_label=$ci_label" >> "$GITHUB_OUTPUT"
echo "has_skip_cicd=$has_skip_cicd" >> "$GITHUB_OUTPUT"
- name: Get changed files
id: changed-files
if: startsWith(github.ref, 'refs/heads/pull-request/')
uses: step-security/changed-files@3dbe17c78367e7d60f00d78ae6781a35be47b4a1 # v45.0.1
with:
base_sha: ${{ steps.base-head-ref.outputs.base_sha }}
files_yaml: |
doc:
- '**.md'
- docs/**
src:
- '!**.md'
- '!docs/**'
- name: Evaluate conditions
id: evaluate
env:
DOCS_ONLY: ${{ steps.changed-files.outputs.doc_any_changed == 'true' && steps.changed-files.outputs.src_any_changed == 'false' }}
CHANGED_DOCS: ${{ steps.changed-files.outputs.doc_all_changed_files }}
CHANGED_SRC: ${{ steps.changed-files.outputs.src_all_changed_files }}
IS_PULLREQUEST: ${{ startsWith(github.ref, 'refs/heads/pull-request/') }}
LABEL: ${{ steps.base-head-ref.outputs.ci_label }}
MERGE_GROUP: ${{ github.event_name == 'merge_group' }}
DISPATCH_TEST_LEVEL: ${{ inputs.test_to_run }}
DISPATCH_IMAGE_TAG: ${{ inputs.image_tag }}
TEST_SCRIPT_PATTERN: ${{ inputs.test_script_pattern }}
TEST_PLATFORM: ${{ inputs.test_platform || 'all' }}
MEGATRON_LM_COMMIT: ${{ inputs.megatron_lm_commit }}
run: |
# Some output that's helpful for debugging
echo "Docs changed: $CHANGED_DOCS"
echo "Src changed: $CHANGED_SRC"
echo "LABEL: $LABEL"
echo "IS_PULLREQUEST: $IS_PULLREQUEST"
echo "DOCS_ONLY: $DOCS_ONLY"
# Run CI only (on main or if label is attached) and if it's not only docs
# Determine test level based on conditions
if [[ "$DOCS_ONLY" == "true" || "$LABEL" == "CI:docs" ]]; then
# For doc-only changes, run only doc tests
TEST_LEVEL="docs"
elif [[ "$LABEL" == "CI:Lfast" ]]; then
TEST_LEVEL="Lfast"
elif [[ "$LABEL" == "CI:L0" ]]; then
TEST_LEVEL="L0"
elif [[ "$LABEL" == "CI:L1" || "$IS_PULLREQUEST" == "false" || "$MERGE_GROUP" == "true" ]]; then
# For labeled PRs, pushes to main (IS_PULL_REQUEST=false), or merge group events, run L1 by default
TEST_LEVEL="L1"
elif [[ "$LABEL" == "CI:L2" ]]; then
TEST_LEVEL="L2"
elif [[ "$LABEL" == "CI:megatron" ]]; then
TEST_LEVEL="L1"
TEST_SCRIPT_PATTERN="megatron"
else
# Skip tests by default for non-labeled PRs
TEST_LEVEL="none"
fi
if [[ "${{ github.event_name }}" == "schedule" ]]; then
echo "Setting test level to L1 for nightly scheduled run"
TEST_LEVEL="L1"
fi
# Override test level if specified in workflow_dispatch
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
echo "Overriding test level from $TEST_LEVEL to $DISPATCH_TEST_LEVEL"
TEST_LEVEL="$DISPATCH_TEST_LEVEL"
fi
if [[ -n "$MEGATRON_LM_COMMIT" && ! "$MEGATRON_LM_COMMIT" =~ ^[0-9a-fA-F]{40}$ ]]; then
echo "megatron_lm_commit must be a full 40-character hexadecimal commit SHA." >&2
exit 1
fi
if [[ -n "$TEST_SCRIPT_PATTERN" && ! "$TEST_SCRIPT_PATTERN" =~ ^[a-zA-Z0-9_.-]+$ ]]; then
echo "test_script_pattern may contain only letters, numbers, dots, underscores, and hyphens." >&2
exit 1
fi
case "$TEST_PLATFORM" in
all | h100 | gb200) ;;
*)
echo "test_platform must be one of: all, h100, gb200." >&2
exit 1
;;
esac
test_plan=$(bash .github/scripts/plan-tests.sh "$TEST_SCRIPT_PATTERN")
echo "Test plan: $test_plan"
unit_test_scripts=$(jq -c '.unit' <<< "$test_plan")
h100_functional_test_scripts=$(jq -c \
'[.functional[] | select((ascii_downcase | contains("gb200")) | not)]' \
<<< "$test_plan")
# The managed Dynamo runtime is currently amd64-only and is not
# installed in the arm64 GB200 image.
gb200_functional_test_scripts=$(jq -c \
'[.functional[] | select(. != "L1_Functional_Tests_Dynamo")]' \
<<< "$test_plan")
unit_test_count=$(jq 'length' <<< "$unit_test_scripts")
h100_functional_test_count=$(jq 'length' <<< "$h100_functional_test_scripts")
gb200_functional_test_count=$(jq 'length' <<< "$gb200_functional_test_scripts")
effective_test_count=0
case "$TEST_LEVEL" in
Lfast)
if [[ "$TEST_PLATFORM" != "gb200" ]]; then
effective_test_count=$((unit_test_count + h100_functional_test_count))
fi
;;
L0)
if [[ "$TEST_PLATFORM" != "gb200" ]]; then
effective_test_count=$unit_test_count
fi
;;
L1 | L2)
if [[ "$TEST_PLATFORM" != "gb200" ]]; then
effective_test_count=$((effective_test_count + unit_test_count + h100_functional_test_count))
fi
if [[ "$TEST_PLATFORM" != "h100" ]]; then
effective_test_count=$((effective_test_count + gb200_functional_test_count))
fi
;;
esac
if [[ "$TEST_LEVEL" != "docs" && "$TEST_LEVEL" != "none" && "$effective_test_count" -eq 0 ]]; then
echo "No test scripts are runnable for test level '$TEST_LEVEL', platform '$TEST_PLATFORM', and pattern '$TEST_SCRIPT_PATTERN'." >&2
exit 1
fi
{
echo "test_level=$TEST_LEVEL"
echo "test_script_pattern=$TEST_SCRIPT_PATTERN"
echo "test_platform=$TEST_PLATFORM"
echo "unit_test_scripts=$unit_test_scripts"
echo "unit_test_count=$unit_test_count"
echo "h100_functional_test_scripts=$h100_functional_test_scripts"
echo "h100_functional_test_count=$h100_functional_test_count"
echo "gb200_functional_test_scripts=$gb200_functional_test_scripts"
echo "gb200_functional_test_count=$gb200_functional_test_count"
echo "megatron_lm_commit=${MEGATRON_LM_COMMIT,,}"
} | tee -a "$GITHUB_OUTPUT"
# Determine image tag: Lfast and docs reuse the main container, workflow_dispatch can override
IMAGE_TAG=""
if [[ "$TEST_LEVEL" == "Lfast" || "$TEST_LEVEL" == "docs" ]]; then
IMAGE_TAG="main"
fi
if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "$DISPATCH_IMAGE_TAG" ]]; then
IMAGE_TAG="$DISPATCH_IMAGE_TAG"
fi
echo "image_tag=$IMAGE_TAG" | tee -a "$GITHUB_OUTPUT"
ephemeral-runner-routing:
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_ephemeral_runner_routing.yml@4fe957d4ea5816af1d67fae130bbab4b434c10b3 # v1.9.2
with:
lane: ${{ vars.NEMO_CI_EPHEMERAL_LANE }}
legacy_label: ${{ vars.NON_NVIDIA_H100_RUNNER }}
run_id: ${{ github.run_id }}
run_attempt: ${{ github.run_attempt }}
org-member-pre-flight:
needs: [ephemeral-runner-routing]
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@d0878a142525ff79366ca96fbc0b306cd8d64670 # v1.9.0
with:
default_runner_prefix: ${{ vars.DEFAULT_H100_RUNNER }}
non_nvidia_runner_prefix: ${{ needs.ephemeral-runner-routing.outputs.runner_label_base }}
default_test_data_path: ${{ vars.DEFAULT_H100_TEST_DATA_PATH }}
non_nvidia_test_data_path: ${{ vars.NON_NVIDIA_H100_TEST_DATA_PATH }}
default_registry: ${{ vars.DEFAULT_H100_CONTAINER_REGISTRY }}
non_nvidia_registry: ${{ vars.NON_NVIDIA_H100_CONTAINER_REGISTRY }}
sso_users_filename: ${{ vars.SSO_USERS_FILENAME }}
secrets:
NVIDIA_MANAGEMENT_ORG_PAT: ${{ secrets.NVIDIA_MANAGEMENT_ORG_PAT }}
gb200-config:
runs-on: ubuntu-latest
outputs:
registry: ${{ steps.config.outputs.registry }}
steps:
- name: Configure GB200 registry
id: config
env:
GB200_REGISTRY: ${{ env.GB200_CONTAINER_REGISTRY }}
run: echo "registry=$GB200_REGISTRY" | tee -a "$GITHUB_OUTPUT"
pr-branch-up-to-date-check:
name: Check if PR branch is up to date
needs: [pre-flight]
if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') }}
runs-on: ubuntu-latest
env:
MAX_COMMITS_BEHIND: 10
steps:
- name: Check how many commits behind target branch
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
BASE_SHA: ${{ needs.pre-flight.outputs.base_sha }}
HEAD_SHA: ${{ needs.pre-flight.outputs.head_sha }}
BASE_REF: ${{ needs.pre-flight.outputs.base_ref }}
HEAD_LABEL: ${{ needs.pre-flight.outputs.head_label }}
run: |
echo "Repository: $REPO"
echo "Base branch: $BASE_REF (SHA: $BASE_SHA)"
echo "PR head: $HEAD_LABEL (SHA: $HEAD_SHA)"
echo "Maximum commits behind allowed: $MAX_COMMITS_BEHIND"
API_RESPONSE=$(gh api "repos/$REPO/compare/$HEAD_SHA...$BASE_REF" --jq '{behind_by: .behind_by, ahead_by: .ahead_by, status: .status}')
COMMITS_BEHIND=$(echo "$API_RESPONSE" | jq -r '.ahead_by')
COMMITS_AHEAD=$(echo "$API_RESPONSE" | jq -r '.behind_by')
STATUS=$(echo "$API_RESPONSE" | jq -r '.status')
echo "Comparison status: $STATUS"
echo "PR is $COMMITS_BEHIND commits behind and $COMMITS_AHEAD commits ahead of $BASE_REF"
# Check if we're behind by more than the allowed number
if [ "$COMMITS_BEHIND" -gt "$MAX_COMMITS_BEHIND" ]; then
echo "❌ ERROR: This PR is $COMMITS_BEHIND commits behind $BASE_REF, which exceeds the maximum allowed ($MAX_COMMITS_BEHIND commits)."
echo "Please rebase or merge the latest changes from $BASE_REF into your PR branch."
exit 1
else
echo "✅ PR is acceptably fresh ($COMMITS_BEHIND commits behind, limit is $MAX_COMMITS_BEHIND)"
fi
lint-check:
name: Lint check
needs: [pre-flight]
runs-on: ubuntu-latest
steps:
- name: Free up disk space
run: |
# Remove unnecessary packages and files on Ubuntu
sudo apt-get clean
sudo rm -rf /usr/local/lib/android || true
sudo rm -rf /opt/ghc || true
sudo rm -rf /usr/local/.ghcup || true
sudo rm -rf /usr/share/dotnet || true
sudo rm -rf /opt/az || true
# Clear pip and npm caches
pip cache purge || true
sudo npm cache clean --force || true
- name: Checkout repository
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
with:
submodules: "recursive"
- name: Install uv
uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
with:
version: "0.11.18"
enable-cache: true
prune-cache: false
# Faster than uv python install since it caches python alongside runner
- name: "Set up Python"
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
with:
python-version-file: ".python-version"
- name: Check lint
run: |
uv venv
uv run --group dev pre-commit install
uv run --group dev pre-commit run --all-files --show-diff-on-failure --color=always
# TODO: this is a temporary check and should be removed once we have 100% correctness
- name: Check if any files with zero errors not in whitelist
run: |
missing_count=0
for file in $(uv run --group dev pyrefly check $(git ls-files 'nemo_rl/**/*.py' 'examples/**/*.py' 'docs/*.py' 'tools/**/*.py') --output-format json | jq -r --slurpfile all_files <(git ls-files 'nemo_rl/**/*.py' 'examples/**/*.py' 'docs/*.py' 'tools/**/*.py' | jq -R -s 'split("\n")[:-1]') --arg pwd "$(pwd)/" '(.errors | group_by(.path) | map({(.[0].path | sub($pwd; "")): length}) | add // {}) as $error_counts | $all_files[0][] | . as $file | if ($error_counts[$file] // 0) == 0 then $file else empty end'); do
if ! fgrep -q "$file" pyrefly.toml; then
echo "File $file has zero errors but is not in pyrefly.toml in the 'project-includes' list. Please add it to this whitelist."
((missing_count++))
fi
done
exit $missing_count
- name: Minimize uv cache
run: uv cache prune --ci
cicd-wait-in-queue:
name: Wait in test approval queue
needs: [pre-flight, lint-check]
runs-on: ubuntu-latest
environment:
name: test
deployment: false
if: >-
${{
always() &&
startsWith(github.ref, 'refs/heads/pull-request/') &&
contains('Lfast L0 L1 L2', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.result == 'success' &&
needs.lint-check.result == 'success' &&
!cancelled()
}}
steps:
- name: Approved
run: echo "Approved to run CI tests."
prepare-megatron-lock:
name: Prepare Megatron lockfile
needs: [pre-flight, cicd-wait-in-queue]
if: >-
${{
always() &&
needs.pre-flight.result == 'success' &&
needs.pre-flight.outputs.megatron_lm_commit != '' &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
!cancelled()
}}
runs-on: ubuntu-latest
outputs:
artifact_name: ${{ steps.package.outputs.artifact_name }}
steps:
- name: Checkout repository
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
with:
ref: ${{ needs.pre-flight.outputs.test_sha }}
submodules: recursive
- name: Override Megatron-LM commit
env:
MEGATRON_LM_COMMIT: ${{ needs.pre-flight.outputs.megatron_lm_commit }}
run: |
megatron_lm_dir="3rdparty/Megatron-Bridge-workspace/Megatron-Bridge/3rdparty/Megatron-LM"
bash .github/scripts/checkout-megatron-lm-commit.sh "$megatron_lm_dir" "$MEGATRON_LM_COMMIT"
- name: Read uv version from Dockerfile
id: uv-version
run: |
uv_version="$(sed -nE 's/^ARG[[:space:]]+UV_VERSION=([^[:space:]#]+).*$/\1/p' docker/Dockerfile)"
if [[ ! "$uv_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "Invalid UV_VERSION from docker/Dockerfile: $uv_version" >&2
exit 1
fi
echo "version=$uv_version" >> "$GITHUB_OUTPUT"
- name: Set up uv
uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
with:
version: ${{ steps.uv-version.outputs.version }}
enable-cache: true
- name: Generate lockfile
run: uv lock
- name: Package lockfile
id: package
env:
MEGATRON_LM_COMMIT: ${{ needs.pre-flight.outputs.megatron_lm_commit }}
NEMO_RL_COMMIT: ${{ needs.pre-flight.outputs.test_sha }}
run: |
artifact_dir="$RUNNER_TEMP/megatron-lock"
megatron_lm_dir="3rdparty/Megatron-Bridge-workspace/Megatron-Bridge/3rdparty/Megatron-LM"
actual_nemo_rl_commit="$(git rev-parse HEAD)"
actual_megatron_lm_commit="$(git -C "$megatron_lm_dir" rev-parse HEAD)"
test "$actual_nemo_rl_commit" = "${NEMO_RL_COMMIT,,}"
test "$actual_megatron_lm_commit" = "${MEGATRON_LM_COMMIT,,}"
mkdir -p "$artifact_dir"
cp uv.lock "$artifact_dir/uv.lock"
{
echo "nemo-rl-commit=$actual_nemo_rl_commit"
echo "megatron-lm-commit=$actual_megatron_lm_commit"
} > "$artifact_dir/manifest.txt"
(
cd "$artifact_dir"
sha256sum uv.lock > uv.lock.sha256
)
echo "artifact_name=megatron-lock-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" >> "$GITHUB_OUTPUT"
- name: Upload lockfile
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: ${{ steps.package.outputs.artifact_name }}
path: ${{ runner.temp }}/megatron-lock
retention-days: 1
if-no-files-found: error
sphinx-build:
needs: [pre-flight, cicd-wait-in-queue]
if: >-
${{
always() &&
needs.pre-flight.result == 'success' &&
contains('docs Lfast L0 L1 L2', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.outputs.test_script_pattern == '' &&
(
needs.cicd-wait-in-queue.result == 'success' ||
needs.pre-flight.outputs.test_level == 'docs' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
!cancelled()
}}
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_docs.yml@5fc4e0babd2a25412bf43fbb526a18b9dd7a11b8 # v0.57.0
build-container:
name: Build H100 container
if: >-
${{
always() &&
needs.pre-flight.result == 'success' &&
needs.org-member-pre-flight.result == 'success' &&
needs.pre-flight.outputs.test_level != 'none' &&
needs.pre-flight.outputs.image_tag == '' &&
(needs.prepare-megatron-lock.result == 'success' || needs.prepare-megatron-lock.result == 'skipped') &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
!cancelled()
}}
needs: [pre-flight, prepare-megatron-lock, org-member-pre-flight, cicd-wait-in-queue]
permissions:
contents: read
pull-requests: read
uses: ./.github/workflows/_build_container.yml
with:
build-ref: ${{ needs.pre-flight.outputs.test_sha }}
image-name: ${{ vars.CI_CONTAINER_NAME }}
dockerfile: docker/Dockerfile
platform: linux/amd64
registry: ${{ needs.org-member-pre-flight.outputs.registry }}
runner: ${{ startsWith(needs.org-member-pre-flight.outputs.runner_prefix, 'ephe-v2-') && format('{0}-a{1}', needs.org-member-pre-flight.outputs.runner_prefix, github.run_attempt) || needs.org-member-pre-flight.outputs.runner_prefix }}
target: release
build-contexts: |
nemo-rl=.
${{ vars.UV_BUILD_CACHE == 'enabled' && format('uv-cache-seed=docker-image://{0}/{1}:uv-cache', needs.org-member-pre-flight.outputs.registry, vars.CI_CONTAINER_NAME) || '' }}
trtllm-ccache-tag: ${{ vars.TRTLLM_BUILD_CACHE == 'enabled' && 'trtllm-ccache' || '' }}
trtllm-wheel-cache-tag: ${{ vars.TRTLLM_BUILD_CACHE == 'enabled' && 'trtllm-wheel-cache' || '' }}
megatron-lm-commit: ${{ needs.pre-flight.outputs.megatron_lm_commit }}
megatron-lock-artifact: ${{ needs.prepare-megatron-lock.outputs.artifact_name }}
build-args: |
MAX_JOBS=4
BUILD_DYNAMO=1
TRTLLM_BUILD_JOBS=24
NEMO_RL_COMMIT=${{ needs.pre-flight.outputs.test_sha }}
build-container-gb200:
name: Build GB200/GCP container
if: >-
${{
always() &&
needs.pre-flight.result == 'success' &&
needs.org-member-pre-flight.result == 'success' &&
needs.gb200-config.result == 'success' &&
needs.pre-flight.outputs.test_level != 'none' &&
needs.pre-flight.outputs.image_tag == '' &&
needs.org-member-pre-flight.outputs.is_member == 'true' &&
vars.DISABLE_GB200_TESTS != 'true' &&
needs.pre-flight.outputs.test_platform != 'h100' &&
contains('L1 L2', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.outputs.gb200_functional_test_count != '0' &&
(needs.prepare-megatron-lock.result == 'success' || needs.prepare-megatron-lock.result == 'skipped') &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
!cancelled()
}}
needs: [pre-flight, prepare-megatron-lock, org-member-pre-flight, gb200-config, cicd-wait-in-queue]
permissions:
contents: read
pull-requests: read
uses: ./.github/workflows/_build_container.yml
with:
build-ref: ${{ needs.pre-flight.outputs.test_sha }}
image-name: ${{ vars.CI_CONTAINER_NAME }}
dockerfile: docker/Dockerfile
platform: linux/arm64
registry: ${{ needs.gb200-config.outputs.registry }}
runner: ${{ vars.GB200_RUNNER }}
target: release
build-contexts: |
nemo-rl=.
${{ vars.UV_BUILD_CACHE == 'enabled' && format('uv-cache-seed=docker-image://{0}/{1}:uv-cache', needs.gb200-config.outputs.registry, vars.CI_CONTAINER_NAME) || '' }}
trtllm-ccache-tag: ${{ vars.TRTLLM_BUILD_CACHE == 'enabled' && 'trtllm-ccache' || '' }}
trtllm-wheel-cache-tag: ${{ vars.TRTLLM_BUILD_CACHE == 'enabled' && 'trtllm-wheel-cache' || '' }}
megatron-lm-commit: ${{ needs.pre-flight.outputs.megatron_lm_commit }}
megatron-lock-artifact: ${{ needs.prepare-megatron-lock.outputs.artifact_name }}
build-args: |
MAX_JOBS=4
TRTLLM_BUILD_JOBS=8
NEMO_RL_COMMIT=${{ needs.pre-flight.outputs.test_sha }}
update-uv-cache:
name: Update uv build cache
needs: [build-container, org-member-pre-flight]
if: >-
${{
github.ref == 'refs/heads/main' &&
vars.UV_BUILD_CACHE == 'enabled' &&
needs.build-container.result == 'success'
}}
runs-on: ${{ startsWith(needs.org-member-pre-flight.outputs.runner_prefix, 'ephe-v2-') && format('{0}-a{1}', needs.org-member-pre-flight.outputs.runner_prefix, github.run_attempt) || needs.org-member-pre-flight.outputs.runner_prefix }}
env:
REGISTRY: ${{ needs.org-member-pre-flight.outputs.registry }}
IMAGE_NAME: ${{ vars.CI_CONTAINER_NAME }}
steps:
- name: Extract and push uv cache image
run: |
set -euo pipefail
SRC="${REGISTRY}/${IMAGE_NAME}:${{ github.run_id }}"
DST="${REGISTRY}/${IMAGE_NAME}:uv-cache"
docker pull "${SRC}"
CID=$(docker create "${SRC}" true)
mkdir -p /tmp/uv-cache
docker cp "${CID}:/opt/nemo_rl_cache/uv/." /tmp/uv-cache/
docker rm "${CID}"
printf 'FROM scratch\nCOPY uv-cache/ /\n' > /tmp/Dockerfile.uv-cache
docker build -t "${DST}" -f /tmp/Dockerfile.uv-cache /tmp
docker push "${DST}"
docker rmi "${SRC}" "${DST}" 2>/dev/null || true
rm -rf /tmp/uv-cache /tmp/Dockerfile.uv-cache
update-uv-cache-gb200:
name: Update GB200 uv build cache
needs: [build-container-gb200, gb200-config]
if: >-
${{
github.ref == 'refs/heads/main' &&
vars.UV_BUILD_CACHE == 'enabled' &&
needs.build-container-gb200.result == 'success'
}}
runs-on: ${{ vars.GB200_RUNNER }}
env:
REGISTRY: ${{ needs.gb200-config.outputs.registry }}
IMAGE_NAME: ${{ vars.CI_CONTAINER_NAME }}
steps:
- name: Extract and push uv cache image
run: |
set -euo pipefail
SRC="${REGISTRY}/${IMAGE_NAME}:${{ github.run_id }}"
DST="${REGISTRY}/${IMAGE_NAME}:uv-cache"
docker pull "${SRC}"
CID=$(docker create "${SRC}" true)
mkdir -p /tmp/uv-cache
docker cp "${CID}:/opt/nemo_rl_cache/uv/." /tmp/uv-cache/
docker rm "${CID}"
printf 'FROM scratch\nCOPY uv-cache/ /\n' > /tmp/Dockerfile.uv-cache
docker build -t "${DST}" -f /tmp/Dockerfile.uv-cache /tmp
docker push "${DST}"
docker rmi "${SRC}" "${DST}" 2>/dev/null || true
rm -rf /tmp/uv-cache /tmp/Dockerfile.uv-cache
cicd-doc-tests:
strategy:
fail-fast: false
matrix:
include:
- script: Docs_Tests
runner: ${{ startsWith(needs.org-member-pre-flight.outputs.runner_prefix, 'ephe-v2-') && format('{0}-a{1}', needs.org-member-pre-flight.outputs.runner_prefix, github.run_attempt) || needs.org-member-pre-flight.outputs.runner_prefix }}
needs: [pre-flight, build-container, org-member-pre-flight, cicd-wait-in-queue]
if: >-
${{
(
always() &&
contains('docs Lfast L0 L1 L2', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.outputs.test_script_pattern == '' &&
needs.pre-flight.result == 'success' &&
needs.org-member-pre-flight.result == 'success' &&
(
needs.cicd-wait-in-queue.result == 'success' ||
needs.pre-flight.outputs.test_level == 'docs' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
(needs.build-container.result == 'success' || needs.build-container.result == 'skipped')
) && !cancelled()
}}
runs-on: ${{ matrix.runner }}
name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }}
steps:
- name: Checkout
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
- name: main
uses: ./.github/actions/test-template
with:
runner: ${{ matrix.runner }}
registry: ${{ needs.org-member-pre-flight.outputs.registry }}
image: ${{ vars.CI_CONTAINER_NAME }}
image-tag: ${{ needs.pre-flight.outputs.image_tag }}
test_data_path: ${{ needs.org-member-pre-flight.outputs.test_data_path }}
script: ${{ matrix.script }}
is_doc_test: "true"
test-commit-sha: ${{ needs.pre-flight.outputs.test_sha }}
cicd-unit-tests:
strategy:
fail-fast: false
max-parallel: 16
matrix:
script: ${{ fromJSON(needs.pre-flight.outputs.unit_test_scripts) }}
needs: [pre-flight, prepare-megatron-lock, build-container, cicd-doc-tests, org-member-pre-flight, cicd-wait-in-queue]
if: >-
${{
(
always() &&
contains('L0 L1 L2 Lfast', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.outputs.test_platform != 'gb200' &&
needs.pre-flight.outputs.unit_test_count != '0' &&
needs.pre-flight.result == 'success' &&
needs.org-member-pre-flight.result == 'success' &&
(needs.prepare-megatron-lock.result == 'success' || needs.prepare-megatron-lock.result == 'skipped') &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
(needs.build-container.result == 'success' || needs.build-container.result == 'skipped') &&
(needs.cicd-doc-tests.result == 'success' || needs.cicd-doc-tests.result == 'skipped')
) && !cancelled()
}}
runs-on: ${{ startsWith(needs.org-member-pre-flight.outputs.runner_prefix, 'ephe-v2-') && format('{0}-a{1}', needs.org-member-pre-flight.outputs.runner_prefix, github.run_attempt) || needs.org-member-pre-flight.outputs.runner_prefix }}
name: ${{ matrix.script }}
steps:
- name: Checkout
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
- name: main
uses: ./.github/actions/test-template
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
with:
runner: ${{ startsWith(needs.org-member-pre-flight.outputs.runner_prefix, 'ephe-v2-') && format('{0}-a{1}', needs.org-member-pre-flight.outputs.runner_prefix, github.run_attempt) || needs.org-member-pre-flight.outputs.runner_prefix }}
script: ${{ matrix.script }}
registry: ${{ needs.org-member-pre-flight.outputs.registry }}
test_data_path: ${{ needs.org-member-pre-flight.outputs.test_data_path }}
image: ${{ vars.CI_CONTAINER_NAME }}
image-tag: ${{ needs.pre-flight.outputs.image_tag }}
is_unit_test: "true"
cpu-only: "false"
test-commit-sha: ${{ needs.pre-flight.outputs.test_sha }}
megatron-lm-commit: ${{ needs.pre-flight.outputs.megatron_lm_commit }}
megatron-lock-artifact: ${{ needs.prepare-megatron-lock.outputs.artifact_name }}
unit-test-script-check:
name: Check unit test script coverage
needs: [pre-flight, cicd-wait-in-queue]
if: >-
${{
always() &&
contains('L0 L1 L2 Lfast', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.result == 'success' &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
!cancelled()
}}
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
with:
ref: ${{ needs.pre-flight.outputs.test_sha }}
- name: Verify unit test planner coverage
run: |
set -euo pipefail
expected=$(mktemp)
planned=$(mktemp)
find tests/unit -maxdepth 1 -type f -name 'L0_Unit*.sh' \
-exec basename {} .sh \; | sort -u > "$expected"
bash .github/scripts/plan-tests.sh "" |
jq -r '.unit[]' | sort -u > "$planned"
missing=$(comm -23 "$expected" "$planned")
if [[ -n "$missing" ]]; then
echo "The following unit scripts are missing from the generated test plan:"
printf '%s\n' "$missing"
exit 1
fi
echo "All L0 unit scripts are included in the generated test plan."
functional-test-script-check:
name: Check functional test script coverage
needs: [pre-flight, cicd-wait-in-queue]
if: >-
${{
always() &&
contains('L1 L2 Lfast', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.result == 'success' &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
!cancelled()
}}
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
with:
ref: ${{ needs.pre-flight.outputs.test_sha }}
- name: Verify functional test planner coverage
run: |
set -euo pipefail
expected=$(mktemp)
planned=$(mktemp)
find tests/functional -maxdepth 1 -type f -name 'L1_Functional*.sh' \
-exec basename {} .sh \; | sort -u > "$expected"
bash .github/scripts/plan-tests.sh "" |
jq -r '.functional[]' | sort -u > "$planned"
missing=$(comm -23 "$expected" "$planned")
if [[ -n "$missing" ]]; then
echo "The following functional scripts are missing from the generated test plan:"
printf '%s\n' "$missing"
exit 1
fi
echo "All L1 functional scripts are included in the generated test plan."
cicd-functional-tests:
strategy:
fail-fast: false
max-parallel: 16
matrix:
script: ${{ fromJSON(needs.pre-flight.outputs.h100_functional_test_scripts) }}
needs: [pre-flight, prepare-megatron-lock, build-container, cicd-unit-tests, functional-test-script-check, org-member-pre-flight, cicd-wait-in-queue]
runs-on: ${{ startsWith(needs.org-member-pre-flight.outputs.runner_prefix, 'ephe-v2-') && format('{0}-a{1}', needs.org-member-pre-flight.outputs.runner_prefix, github.run_attempt) || needs.org-member-pre-flight.outputs.runner_prefix }}
if: >-
${{
always() &&
contains('L1 L2', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.outputs.test_platform != 'gb200' &&
needs.pre-flight.outputs.h100_functional_test_count != '0' &&
needs.pre-flight.result == 'success' &&
needs.org-member-pre-flight.result == 'success' &&
(needs.prepare-megatron-lock.result == 'success' || needs.prepare-megatron-lock.result == 'skipped') &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
(needs.build-container.result == 'success' || needs.build-container.result == 'skipped') &&
(
needs.cicd-unit-tests.result == 'success' ||
(
(needs.pre-flight.outputs.test_platform == 'gb200' || needs.pre-flight.outputs.unit_test_count == '0') &&
needs.cicd-unit-tests.result == 'skipped'
)
) &&
needs.functional-test-script-check.result == 'success' &&
!cancelled()
}}
name: ${{ matrix.script }}
steps:
- name: Checkout
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
- name: main
uses: ./.github/actions/test-template
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
with:
runner: ${{ startsWith(needs.org-member-pre-flight.outputs.runner_prefix, 'ephe-v2-') && format('{0}-a{1}', needs.org-member-pre-flight.outputs.runner_prefix, github.run_attempt) || needs.org-member-pre-flight.outputs.runner_prefix }}
registry: ${{ needs.org-member-pre-flight.outputs.registry }}
image: ${{ vars.CI_CONTAINER_NAME }}
test_data_path: ${{ needs.org-member-pre-flight.outputs.test_data_path }}
script: ${{ matrix.script }}
test-commit-sha: ${{ needs.pre-flight.outputs.test_sha }}
megatron-lm-commit: ${{ needs.pre-flight.outputs.megatron_lm_commit }}
megatron-lock-artifact: ${{ needs.prepare-megatron-lock.outputs.artifact_name }}
cicd-functional-tests-gb200:
strategy:
fail-fast: false
max-parallel: 16
matrix:
script: ${{ fromJSON(needs.pre-flight.outputs.gb200_functional_test_scripts) }}
needs: [pre-flight, prepare-megatron-lock, build-container-gb200, cicd-unit-tests, functional-test-script-check, org-member-pre-flight, gb200-config, cicd-wait-in-queue]
runs-on: ${{ vars.GB200_RUNNER }}
if: >-
${{
always() &&
contains('L1 L2', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.outputs.test_platform != 'h100' &&
needs.pre-flight.outputs.gb200_functional_test_count != '0' &&
needs.org-member-pre-flight.outputs.is_member == 'true' &&
vars.DISABLE_GB200_TESTS != 'true' &&
needs.pre-flight.result == 'success' &&
needs.org-member-pre-flight.result == 'success' &&
(needs.prepare-megatron-lock.result == 'success' || needs.prepare-megatron-lock.result == 'skipped') &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
(needs.build-container-gb200.result == 'success' || needs.build-container-gb200.result == 'skipped') &&
(
needs.cicd-unit-tests.result == 'success' ||
(
(needs.pre-flight.outputs.test_platform == 'gb200' || needs.pre-flight.outputs.unit_test_count == '0') &&
needs.cicd-unit-tests.result == 'skipped'
)
) &&
needs.functional-test-script-check.result == 'success' &&
!cancelled()
}}
name: gb200_${{ matrix.script }}
steps:
- name: Checkout
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
- name: main
uses: ./.github/actions/test-template
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
with:
runner: ${{ vars.GB200_RUNNER }}
registry: ${{ needs.gb200-config.outputs.registry }}
image: ${{ vars.CI_CONTAINER_NAME }}
test_data_path: ${{ needs.org-member-pre-flight.outputs.test_data_path }}
image-tag: ${{ needs.pre-flight.outputs.image_tag }}
script: ${{ matrix.script }}
test-commit-sha: ${{ needs.pre-flight.outputs.test_sha }}
megatron-lm-commit: ${{ needs.pre-flight.outputs.megatron_lm_commit }}
megatron-lock-artifact: ${{ needs.prepare-megatron-lock.outputs.artifact_name }}
cicd-fast-functional-tests:
strategy:
fail-fast: false
matrix:
script: ${{ fromJSON(needs.pre-flight.outputs.h100_functional_test_scripts) }}
needs: [pre-flight, prepare-megatron-lock, functional-test-script-check, org-member-pre-flight, cicd-wait-in-queue]
if: >-
${{
always() &&
contains('Lfast', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.outputs.test_platform != 'gb200' &&
needs.pre-flight.outputs.h100_functional_test_count != '0' &&
needs.pre-flight.result == 'success' &&
needs.org-member-pre-flight.result == 'success' &&
(needs.prepare-megatron-lock.result == 'success' || needs.prepare-megatron-lock.result == 'skipped') &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
needs.functional-test-script-check.result == 'success' &&
!cancelled()
}}
runs-on: ${{ startsWith(needs.org-member-pre-flight.outputs.runner_prefix, 'ephe-v2-') && format('{0}-a{1}', needs.org-member-pre-flight.outputs.runner_prefix, github.run_attempt) || needs.org-member-pre-flight.outputs.runner_prefix }}
name: fast_${{ matrix.script }}
steps:
- name: Checkout
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
- name: main
uses: ./.github/actions/test-template
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
with:
runner: ${{ startsWith(needs.org-member-pre-flight.outputs.runner_prefix, 'ephe-v2-') && format('{0}-a{1}', needs.org-member-pre-flight.outputs.runner_prefix, github.run_attempt) || needs.org-member-pre-flight.outputs.runner_prefix }}
script: ${{ matrix.script }}
image-tag: ${{ needs.pre-flight.outputs.image_tag }}
registry: ${{ needs.org-member-pre-flight.outputs.registry }}
image: ${{ vars.CI_CONTAINER_NAME }}
test_data_path: ${{ needs.org-member-pre-flight.outputs.test_data_path }}
test-commit-sha: ${{ needs.pre-flight.outputs.test_sha }}
megatron-lm-commit: ${{ needs.pre-flight.outputs.megatron_lm_commit }}
megatron-lock-artifact: ${{ needs.prepare-megatron-lock.outputs.artifact_name }}
CI_QA_Gate:
name: "CI quality check${{ needs.pre-flight.outputs.test_level == 'none' && ' (No tests run: Label CI:L*)' || '' }}"
if: always()
runs-on: ubuntu-latest
needs:
- pre-flight
- org-member-pre-flight
- pr-branch-up-to-date-check
- lint-check
- cicd-wait-in-queue
- prepare-megatron-lock
- sphinx-build
- build-container
- build-container-gb200
- cicd-doc-tests
- cicd-unit-tests
- unit-test-script-check
- functional-test-script-check
- cicd-functional-tests
- cicd-functional-tests-gb200
- cicd-fast-functional-tests
steps:
- name: main
env:
JOB_RESULTS: ${{ toJSON(needs) }}
# Job is considered successful if nothing was run, or if all jobs were successful (the tests run even if only docs were run b/c doctests are selected)
ALL_SUCCESS: >-
${{
needs.lint-check.result == 'success' &&
(needs.cicd-wait-in-queue.result == 'success' || needs.cicd-wait-in-queue.result == 'skipped') &&
(needs.prepare-megatron-lock.result == 'success' || needs.prepare-megatron-lock.result == 'skipped') &&
(needs.pr-branch-up-to-date-check.result == 'success' || needs.pr-branch-up-to-date-check.result == 'skipped') &&
(
needs.pre-flight.outputs.test_level != 'none' &&
(
needs.sphinx-build.result == 'success' ||
(needs.pre-flight.outputs.test_script_pattern != '' && needs.sphinx-build.result == 'skipped')
) &&
(needs.build-container.result == 'success' || needs.build-container.result == 'skipped') &&
(needs.build-container-gb200.result == 'success' || needs.build-container-gb200.result == 'skipped') &&
(
(
(needs.cicd-doc-tests.result == 'success' || needs.cicd-doc-tests.result == 'skipped') &&
(
!contains('L0 L1 L2 Lfast', needs.pre-flight.outputs.test_level) ||
needs.pre-flight.outputs.test_platform == 'gb200' ||
needs.pre-flight.outputs.unit_test_count == '0' ||
needs.cicd-unit-tests.result == 'success'
) &&
(
!contains('L0 L1 L2 Lfast', needs.pre-flight.outputs.test_level) ||
needs.unit-test-script-check.result == 'success'
) &&
(
!contains('L1 L2 Lfast', needs.pre-flight.outputs.test_level) ||
needs.functional-test-script-check.result == 'success'
) &&
(
!contains('L1 L2', needs.pre-flight.outputs.test_level) ||
needs.pre-flight.outputs.test_platform == 'gb200' ||
needs.pre-flight.outputs.h100_functional_test_count == '0' ||
needs.cicd-functional-tests.result == 'success'
) &&
(
(
needs.pre-flight.outputs.test_platform != 'gb200' &&
(
needs.org-member-pre-flight.outputs.is_member != 'true' ||
vars.DISABLE_GB200_TESTS == 'true'
)
) ||
needs.pre-flight.outputs.test_platform == 'h100' ||
!contains('L1 L2', needs.pre-flight.outputs.test_level) ||
needs.pre-flight.outputs.gb200_functional_test_count == '0' ||
needs.cicd-functional-tests-gb200.result == 'success'
) &&
(
!contains('Lfast', needs.pre-flight.outputs.test_level) ||
needs.pre-flight.outputs.test_platform == 'gb200' ||
needs.pre-flight.outputs.h100_functional_test_count == '0' ||
needs.cicd-fast-functional-tests.result == 'success'
)
)
)
)
}}
CI_SKIP: ${{ needs.pre-flight.outputs.has_skip_cicd }}
TEST_LEVEL: ${{ needs.pre-flight.outputs.test_level }}
run: |
SUMMARY=$(echo $JOB_RESULTS | jq 'to_entries[] | .key + ": " + .value.result' | tr -d '"')
echo '🤖: CICD Result for test level: ${{ needs.pre-flight.outputs.test_level }}' >> $GITHUB_STEP_SUMMARY
echo "$SUMMARY" >> $GITHUB_STEP_SUMMARY
if [[ "$TEST_LEVEL" == "none" ]]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "---" >> $GITHUB_STEP_SUMMARY
echo "⚠️ **No tests were run.** This PR does not have a CI label." >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "To trigger tests, add one of the following labels to your PR:" >> $GITHUB_STEP_SUMMARY
echo "| Label | What it runs |" >> $GITHUB_STEP_SUMMARY
echo "|-------|-------------|" >> $GITHUB_STEP_SUMMARY
echo "| \`CI:docs\` | Doc tests only |" >> $GITHUB_STEP_SUMMARY
echo "| \`CI:Lfast\` | Fast subset (reuses main container) |" >> $GITHUB_STEP_SUMMARY
echo "| \`CI:L0\` | Unit tests + docs + lint |" >> $GITHUB_STEP_SUMMARY
echo "| \`CI:L1\` | L0 + functional tests |" >> $GITHUB_STEP_SUMMARY
echo "| \`CI:L2\` | L1 + convergence tests |" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "This check will remain failed until a CI label is added." >> $GITHUB_STEP_SUMMARY
fi
test "$ALL_SUCCESS" = "true" || test "$CI_SKIP" = "true"
notify-nightly-failure:
name: Notify nightly test failure
runs-on: ubuntu-latest
needs: [CI_QA_Gate]
environment:
name: main
deployment: false
if: ${{ always() && github.event_name == 'schedule' && needs.CI_QA_Gate.result == 'failure' }}
steps:
- name: Send Slack notification
env:
SLACK_WEBHOOK: ${{ secrets.SLACK_TEAM_CHANNEL_WEBHOOK }}
run: |
MESSAGE='{
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "🚨 Nightly GitHub CI test failed on main branch\n\n• Repository: ${{ github.repository }}\n• Commit: `${{ github.sha }}`\n• Workflow: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"
}
}
]
}'
curl -X POST -H "Content-type: application/json" --data "$MESSAGE" "$SLACK_WEBHOOK"
Coverage:
runs-on: ubuntu-latest
needs:
- CI_QA_Gate
- cicd-doc-tests
- cicd-unit-tests
- cicd-functional-tests
if: always()
strategy:
matrix:
flag: [doc-test, unit-test, e2e]
steps:
- name: Get PR info
id: get-pr-info
if: startsWith(github.ref, 'refs/heads/pull-request/')
uses: nv-gha-runners/get-pr-info@090577647b8ddc4e06e809e264f7881650ecdccf # main
- name: Checkout
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
- name: Download coverage reports of current branch
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
with:
pattern: coverage-${{ matrix.flag }}-*
path: coverage-downloads
- name: Check if artifacts were downloaded
id: check-artifacts
run: |
# Check if any .coverage files were downloaded
if find coverage-downloads -name ".coverage" -type f 2>/dev/null | grep -q .; then
echo "artifacts-found=true" >> $GITHUB_OUTPUT
echo "Found coverage artifacts for ${{ matrix.flag }}"
else
echo "artifacts-found=false" >> $GITHUB_OUTPUT
echo "No coverage artifacts found for ${{ matrix.flag }}"
fi
- name: Get total coverage of current branch
shell: bash -x -e -u -o pipefail {0}
if: ${{ steps.check-artifacts.outputs.artifacts-found == 'true' }}
run: |
pip install coverage
find coverage-downloads -name ".coverage" -type f
coverage combine --keep $(find coverage-downloads -name ".coverage" -type f)
coverage report -i --show-missing
rm -rf coverage-downloads
- name: Skip coverage processing
if: ${{ steps.check-artifacts.outputs.artifacts-found == 'false' }}
run: |
echo "No coverage artifacts found for ${{ matrix.flag }}, skipping coverage processing"
- name: Upload coverage reports to Codecov
if: ${{ steps.check-artifacts.outputs.artifacts-found == 'true' }}
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
verbose: true
flags: ${{ matrix.flag }}
# Copied PR branches run on push, so upload-coverage needs explicit PR metadata.
override_pr: ${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number || '' }}
commit_parent: ${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').base.sha || '' }}
- name: Upload artifacts
if: ${{ steps.check-artifacts.outputs.artifacts-found == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: coverage-${{ matrix.flag }}-aggregated
path: |
.coverage
include-hidden-files: true
DCO_merge_group:
name: DCO
if: github.event_name == 'merge_group'
runs-on: ubuntu-latest
steps:
- run: echo "The real DCO check happens on PRs only. This is a placeholder for the merge queue to keep the DCO check as a required status check."