Repository navigation

Nightly CI Pipeline #271

Workflow file for this run

.github/workflows/nightly-ci.yml at 0fac8e8

	# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
	# SPDX-License-Identifier: Apache-2.0

	name: Nightly CI Pipeline

	on:
	schedule:
	- cron: '0 8 * * *' # Every day at 12:00 AM PST (08:00 UTC)
	workflow_dispatch: # Allow manual triggering for testing
	inputs:
	release:
	description: 'Stage NGC images, Artifactory wheels, and trigger the GitLab release pipeline. Schedule always sets release=true.'
	required: false
	type: boolean
	default: false
	run_tests:
	description: 'Run vllm/sglang/trtllm tests + dynamo-pipeline checks. Schedule always runs them.'
	required: false
	type: boolean
	default: true
	skip_gitlab_pipeline:
	description: 'Skip the GitLab release automation pipeline trigger. Emergency use only.'
	required: false
	type: boolean
	default: false

	permissions:
	contents: read
	# Lets the build jobs' compliance-extract query the Actions API for the
	# previous successful cron-triggered nightly run (manual runs are excluded)
	# and download its baseline compliance artifact to diff the OSRB CSV against.
	# A reusable workflow's token can't exceed the caller's.
	actions: read


	env:
	BUILDER_NAME: b-${{ github.run_id }}-${{ github.run_attempt }}

	jobs:

	# ============================================================================
	# PRE-WARM K8S BUILDER
	# ============================================================================
	create-fresh-builder:
	name: Create fresh K8s builder
	runs-on: prod-default-small-v2
	permissions:
	contents: read
	outputs:
	builder_name: ${{ steps.export-builder-name.outputs.builder_name }}
	steps:
	- name: Checkout repository
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	- name: Export builder name
	id: export-builder-name
	run: \|
	echo "builder_name=${{ env.BUILDER_NAME }}" >> $GITHUB_OUTPUT
	- name: Create and bootstrap fresh K8s builder
	uses: $/.github/actions/bootstrap-buildkit
	with:
	builder_name: ${{ steps.export-builder-name.outputs.builder_name }}
	buildkit_worker_addresses: ''
	namespace: ${{ vars.BUILDKIT_NAMESPACE }}
	suppress_fallback_warning: 'true'

	# ============================================================================
	# RESOLVE SOURCE SHA
	# ============================================================================
	# Single SHA every downstream job (shared-build-image, dynamo-pipeline,
	# shared-test, release.yml) builds, tags, and releases against. On schedule
	# and workflow_dispatch from main, this is just github.sha.
	resolve-source-sha:
	name: Resolve source SHA
	runs-on: prod-default-v2
	permissions:
	contents: read
	outputs:
	source_sha: ${{ steps.resolve.outputs.source_sha }}
	steps:
	- id: resolve
	shell: bash
	run: \|
	set -euo pipefail
	SHA="${GITHUB_SHA}"
	echo "Using caller SHA: ${SHA}"
	echo "source_sha=${SHA}" >> "$GITHUB_OUTPUT"

	# ============================================================================
	# COMPUTE NIGHTLY DEV VERSION
	# ============================================================================
	# Emits a PEP 440 dev suffix (e.g. .dev20260423) forwarded to every
	# pipeline below. At the leaf, the suffix is stamped into pyproject / Cargo
	# versions on the runner before docker build, so wheels produced by the
	# wheel_builder stage carry the dev version.
	compute-dev-version:
	name: Compute dev version suffix
	runs-on: prod-default-v2
	permissions:
	contents: read
	outputs:
	dev_suffix: ${{ steps.compute.outputs.dev_suffix }}
	steps:
	- id: compute
	shell: bash
	run: \|
	DATE=$(date -u +%Y%m%d)
	echo "dev_suffix=.dev${DATE}" >> $GITHUB_OUTPUT

	# ============================================================================
	# COMPUTE RELEASE MODE
	# ============================================================================
	# schedule → always release. workflow_dispatch → honor the `release` input.
	# Output is consumed by the `release` job below to gate the workflow_call
	# into release.yml.
	compute-release-mode:
	name: Compute release mode
	runs-on: prod-default-v2
	permissions:
	contents: read
	outputs:
	release: ${{ steps.compute.outputs.release }}
	run_tests: ${{ steps.compute.outputs.run_tests }}
	steps:
	- id: compute
	shell: bash
	env:
	DISPATCH_RELEASE: ${{ inputs.release }}
	DISPATCH_RUN_TESTS: ${{ inputs.run_tests }}
	run: \|
	case "${GITHUB_EVENT_NAME}" in
	schedule)
	# cron must keep tests on so a failing nightly blocks the release.
	echo "release=true" >> "$GITHUB_OUTPUT"
	echo "run_tests=true" >> "$GITHUB_OUTPUT"
	;;
	workflow_dispatch)
	if [ "${DISPATCH_RELEASE}" = "false" ]; then
	echo "release=false" >> "$GITHUB_OUTPUT"
	else
	echo "release=true" >> "$GITHUB_OUTPUT"
	fi
	if [ "${DISPATCH_RUN_TESTS}" = "false" ]; then
	echo "run_tests=false" >> "$GITHUB_OUTPUT"
	else
	echo "run_tests=true" >> "$GITHUB_OUTPUT"
	fi
	;;
	*)
	echo "release=false" >> "$GITHUB_OUTPUT"
	echo "run_tests=true" >> "$GITHUB_OUTPUT"
	;;
	esac

	# Seeds the nightly Slack thread; exposes ts so downstream jobs (and the
	# GitLab pipeline) can reply. continue-on-error so a Slack outage doesn't
	# block the nightly — downstream jobs skip on empty outputs.
	notify-slack-start:
	name: Notify Slack — nightly started
	runs-on: prod-default-v2
	needs: [compute-release-mode, resolve-source-sha, compute-dev-version]
	if: ${{ needs.compute-release-mode.outputs.release == 'true' }}
	permissions:
	contents: read
	outputs:
	thread_ts: ${{ steps.post.outputs.thread_ts }}
	steps:
	- name: Post start message
	id: post
	continue-on-error: true
	env:
	SLACK_BOT_TOKEN: ${{ secrets.SLACK_RELEASE_BOT_TOKEN }}
	SLACK_CHANNEL_ID: ${{ secrets.SLACK_RELEASE_CHANNEL_ID }}
	SOURCE_SHA: ${{ needs.resolve-source-sha.outputs.source_sha }}
	DEV_SUFFIX: ${{ needs.compute-dev-version.outputs.dev_suffix }}
	RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
	shell: bash
	run: \|
	set -euo pipefail
	DATE_UTC=$(date -u +%Y-%m-%d)
	SHORT_SHA="${SOURCE_SHA:0:7}"

	PAYLOAD=$(jq -n \
	--arg channel "${SLACK_CHANNEL_ID}" \
	--arg date "${DATE_UTC}" \
	--arg sha "${SHORT_SHA}" \
	--arg run_url "${RUN_URL}" \
	--arg dev "${DEV_SUFFIX}" \
	'{
	channel: $channel,
	text: ("Dynamo Nightly build started — " + $date),
	blocks: [
	{ type: "header",
	text: { type: "plain_text", text: (":crescent_moon: Dynamo Nightly build started — " + $date) } },
	{ type: "section",
	fields: [
	{ type: "mrkdwn", text: ("Commit:\n`" + $sha + "`") },
	{ type: "mrkdwn", text: ("Dev version suffix:\n`" + $dev + "`") }
	] },
	{ type: "context",
	elements: [ { type: "mrkdwn", text: ("<" + $run_url + "\|GitHub Actions run>") } ] }
	]
	}')

	RESPONSE=$(curl -fsSL -X POST https://slack.com/api/chat.postMessage \
	-H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \
	-H "Content-type: application/json; charset=utf-8" \
	--data "${PAYLOAD}")

	OK=$(echo "${RESPONSE}" \| jq -r '.ok')
	if [ "${OK}" != "true" ]; then
	# RESPONSE may carry channel/ts/warning fields; surface only .error.
	ERR=$(echo "${RESPONSE}" \| jq -r '.error // "unknown"')
	echo "::error::Slack chat.postMessage failed: ${ERR}"
	exit 1
	fi
	TS=$(echo "${RESPONSE}" \| jq -r '.ts')
	# ts is a runtime per-message identifier (not a secret) and is the only
	# value other jobs need; channel ID is read by every downstream Slack
	# step directly from secrets.SLACK_RELEASE_CHANNEL_ID so it stays
	# auto-masked. Do NOT ::add-mask:: the ts — masking it before writing
	# GITHUB_OUTPUT caused the value to arrive empty across the
	# workflow_call boundary, silently skipping downstream reply steps.
	echo "thread_ts=${TS}" >> "${GITHUB_OUTPUT}"
	echo "Posted nightly start message"

	# ============================================================================
	# MANUAL RELEASE APPROVAL GATE
	# ============================================================================
	# Manual workflow_dispatch that would publish (release=true) must be approved
	# by a reviewer of the `manual-release-approval` environment before the
	# `release` job runs release.yml. Scheduled runs skip this job; the `release`
	# job below accepts `skipped` as success.
	manual-release-approval:
	name: Manual release approval gate
	needs: [compute-release-mode]
	if: ${{ github.event_name == 'workflow_dispatch' && needs.compute-release-mode.outputs.release == 'true' }}
	runs-on: prod-default-small-v2
	environment: manual-release-approval
	permissions:
	contents: read
	steps:
	- name: Record approval
	run: \|
	echo "Manual nightly release approved by ${{ github.actor }}"
	echo "Run: ${{ github.run_id }} attempt ${{ github.run_attempt }}"

	# ============================================================================
	# BUILD JOBS
	# ============================================================================

	vllm-build:
	name: vllm-runtime # This name overlaps with other vllm jobs to group them in the UI
	needs: [create-fresh-builder, resolve-source-sha, compute-dev-version]
	uses: $/.github/workflows/shared-build-image.yml
	with:
	framework: vllm
	target: runtime
	cuda_version: '["13.0"]'
	platform: 'linux/amd64,linux/arm64'
	builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }}
	inline_compliance: true
	build_timeout_minutes: 120
	# Source archival temporarily disabled until the wheel-builder build
	# context includes deploy/inference-gateway/ext-proc (a cargo vendor
	# workspace member).
	archive_sources: false
	# OSRB CSV diff baseline: nightly context -> diff against the previous
	# successful scheduled nightly run's compliance artifacts; manual runs
	# are excluded (see resolve_diff_base.py).
	diff_event_context: nightly
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	# Stamp the checkout with the shared nightly dev suffix BEFORE the build
	# (apply_dev_version.py in shared-build-image), so the wheels embedded in
	# the image report X.Y.Z.devYYYYMMDD — matching the Artifactory-published
	# nightly wheels and the dynamo-runtime image from dynamo-pipeline.
	dev_version_suffix: ${{ needs.compute-dev-version.outputs.dev_suffix }}
	secrets: inherit

	sglang-build:
	name: sglang-runtime # This name overlaps with other sglang jobs to group them in the UI
	needs: [create-fresh-builder, resolve-source-sha, compute-dev-version]
	uses: $/.github/workflows/shared-build-image.yml
	with:
	framework: sglang
	target: runtime
	cuda_version: '["13.0"]'
	platform: 'linux/amd64,linux/arm64'
	builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }}
	inline_compliance: true
	build_timeout_minutes: 120
	archive_sources: false
	# OSRB CSV diff baseline: nightly context -> diff against the previous
	# successful scheduled nightly run's compliance artifacts; manual runs
	# are excluded (see resolve_diff_base.py).
	diff_event_context: nightly
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	# Stamp the checkout with the shared nightly dev suffix BEFORE the build
	# (apply_dev_version.py in shared-build-image), so the wheels embedded in
	# the image report X.Y.Z.devYYYYMMDD — matching the Artifactory-published
	# nightly wheels and the dynamo-runtime image from dynamo-pipeline.
	dev_version_suffix: ${{ needs.compute-dev-version.outputs.dev_suffix }}
	secrets: inherit

	trtllm-build:
	name: trtllm-runtime # This name overlaps with other trtllm jobs to group them in the UI
	needs: [create-fresh-builder, resolve-source-sha, compute-dev-version]
	uses: $/.github/workflows/shared-build-image.yml
	with:
	framework: trtllm
	target: runtime
	cuda_version: '["13.1"]'
	platform: 'linux/amd64,linux/arm64'
	builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }}
	inline_compliance: true
	build_timeout_minutes: 120
	archive_sources: false
	# OSRB CSV diff baseline: nightly context -> diff against the previous
	# successful scheduled nightly run's compliance artifacts; manual runs
	# are excluded (see resolve_diff_base.py).
	diff_event_context: nightly
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	# Stamp the checkout with the shared nightly dev suffix BEFORE the build
	# (apply_dev_version.py in shared-build-image), so the wheels embedded in
	# the image report X.Y.Z.devYYYYMMDD — matching the Artifactory-published
	# nightly wheels and the dynamo-runtime image from dynamo-pipeline.
	dev_version_suffix: ${{ needs.compute-dev-version.outputs.dev_suffix }}
	secrets: inherit

	# ============================================================================
	# OPTIONAL NIGHTLY IMAGES (operator, planner, frontend, runtime EFA
	# variants)
	# ============================================================================
	# Built HERE — not reused from post-merge — for two reasons:
	# 1. OSRB from the get-go: GitLab's nvbug:attach-compliance resolves
	# compliance-<sha>-<token> artifacts on the run id this workflow
	# forwards, so the CSVs must be produced by THIS run.
	# 2. Tag isolation (same invariant as the runtime builds): the -nightly
	# ECR tag suffix means a missing or failed nightly build can never
	# fall through to — or overwrite — the post-merge image at the same SHA.
	# None of these jobs gates the release below: release.yml copies each
	# fail-soft (build-result gate + crane-manifest probe, warn-skip) so a
	# flake in any optional image cannot block the runtime nightly.
	# resolve-source-sha is always GITHUB_SHA here, so github.sha-keyed tags
	# match the resolved source.

	operator-build:
	name: kubernetes-operator
	needs: [create-fresh-builder, resolve-source-sha]
	runs-on: prod-default-v2
	outputs:
	operator_default_tag: ${{ steps.build.outputs.image_tag }}
	permissions:
	contents: read
	actions: read
	steps:
	- name: Checkout code
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	# Do not use fetch-depth: 0 — it fetches all 1600+ branches (~3 min)
	- name: Build and push operator (nightly tag)
	id: build
	uses: $/.github/actions/build-deploy-component
	with:
	component: operator
	image_tag: ${{ github.sha }}-operator-nightly
	# Canonical compliance artifact name — OSRB attach tokens and the
	# nightly diff-baseline lookup key on compliance-<sha>-operator,
	# independent of the -nightly image-tag suffix.
	artifact_tag: ${{ github.sha }}-operator
	builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }}
	namespace: ${{ vars.BUILDKIT_NAMESPACE }}
	aws_default_region: ${{ vars.AWS_DEFAULT_REGION }}
	ecr_registry: ${{ vars.ECR_REGISTRY }}
	ecr_repository: ${{ vars.ECR_REPOSITORY }}
	azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
	azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
	azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
	extra_tags: \|
	${{ secrets.AZURE_ACR_HOSTNAME }}/${{ vars.ACR_REPOSITORY }}:${{ github.sha }}-operator-nightly
	# OSRB CSV diff baseline: previous successful scheduled nightly run
	# (first nightly soft-degrades to baseline_unavailable).
	diff_event_context: nightly
	gh_token: ${{ secrets.GITHUB_TOKEN }}

	planner-build:
	name: dynamo-planner
	needs: [create-fresh-builder, resolve-source-sha, compute-dev-version]
	uses: $/.github/workflows/shared-build-image.yml
	with:
	framework: dynamo
	target: planner
	cuda_version: '[""]'
	platform: 'linux/amd64,linux/arm64'
	builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }}
	inline_compliance: true
	build_timeout_minutes: 45
	archive_sources: false
	# OSRB CSV diff baseline: nightly context -> diff against the previous
	# successful scheduled nightly run's compliance artifacts; manual runs
	# are excluded (see resolve_diff_base.py). Compliance artifact is
	# compliance-<sha>-dynamo-planner (target_tag_plain — no -nightly
	# suffix), matching the OSRB token.
	diff_event_context: nightly
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	# Stamp the checkout with the shared nightly dev suffix BEFORE the build
	# (apply_dev_version.py in shared-build-image), so the wheels embedded in
	# the image report X.Y.Z.devYYYYMMDD — matching the Artifactory-published
	# nightly wheels and the dynamo-runtime image from dynamo-pipeline.
	dev_version_suffix: ${{ needs.compute-dev-version.outputs.dev_suffix }}
	secrets: inherit

	frontend-build:
	name: dynamo-frontend
	needs: [create-fresh-builder, resolve-source-sha, compute-dev-version]
	uses: $/.github/workflows/shared-build-image.yml
	with:
	framework: dynamo
	target: frontend
	cuda_version: '[""]'
	platform: 'linux/amd64,linux/arm64'
	builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }}
	inline_compliance: true
	build_timeout_minutes: 45
	archive_sources: false
	# Compliance artifact is compliance-<sha>-dynamo-frontend
	# (target_tag_plain — no -nightly suffix), matching the OSRB token.
	diff_event_context: nightly
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	# Stamp the checkout with the shared nightly dev suffix BEFORE the build
	# (apply_dev_version.py in shared-build-image), so the wheels embedded in
	# the image report X.Y.Z.devYYYYMMDD — matching the Artifactory-published
	# nightly wheels and the dynamo-runtime image from dynamo-pipeline.
	dev_version_suffix: ${{ needs.compute-dev-version.outputs.dev_suffix }}
	secrets: inherit

	vllm-efa-build:
	name: vllm-runtime-efa
	needs: [create-fresh-builder, resolve-source-sha, compute-dev-version]
	uses: $/.github/workflows/shared-build-image.yml
	with:
	framework: vllm
	target: runtime
	cuda_version: '["13.0"]'
	platform: 'linux/amd64,linux/arm64'
	make_efa: true
	builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }}
	inline_compliance: true
	build_timeout_minutes: 120
	archive_sources: false
	diff_event_context: nightly
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	# Stamp the checkout with the shared nightly dev suffix BEFORE the build
	# (apply_dev_version.py in shared-build-image), so the wheels embedded in
	# the image report X.Y.Z.devYYYYMMDD — matching the Artifactory-published
	# nightly wheels and the dynamo-runtime image from dynamo-pipeline.
	dev_version_suffix: ${{ needs.compute-dev-version.outputs.dev_suffix }}
	secrets: inherit

	sglang-efa-build:
	name: sglang-runtime-efa
	needs: [create-fresh-builder, resolve-source-sha, compute-dev-version]
	uses: $/.github/workflows/shared-build-image.yml
	with:
	framework: sglang
	target: runtime
	cuda_version: '["13.0"]'
	platform: 'linux/amd64,linux/arm64'
	make_efa: true
	builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }}
	inline_compliance: true
	build_timeout_minutes: 120
	archive_sources: false
	diff_event_context: nightly
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	# Stamp the checkout with the shared nightly dev suffix BEFORE the build
	# (apply_dev_version.py in shared-build-image), so the wheels embedded in
	# the image report X.Y.Z.devYYYYMMDD — matching the Artifactory-published
	# nightly wheels and the dynamo-runtime image from dynamo-pipeline.
	dev_version_suffix: ${{ needs.compute-dev-version.outputs.dev_suffix }}
	secrets: inherit

	trtllm-efa-build:
	name: trtllm-runtime-efa
	needs: [create-fresh-builder, resolve-source-sha, compute-dev-version]
	uses: $/.github/workflows/shared-build-image.yml
	with:
	framework: trtllm
	target: runtime
	cuda_version: '["13.1"]'
	platform: 'linux/amd64,linux/arm64'
	make_efa: true
	builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }}
	inline_compliance: true
	build_timeout_minutes: 120
	archive_sources: false
	diff_event_context: nightly
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	# Stamp the checkout with the shared nightly dev suffix BEFORE the build
	# (apply_dev_version.py in shared-build-image), so the wheels embedded in
	# the image report X.Y.Z.devYYYYMMDD — matching the Artifactory-published
	# nightly wheels and the dynamo-runtime image from dynamo-pipeline.
	dev_version_suffix: ${{ needs.compute-dev-version.outputs.dev_suffix }}
	secrets: inherit

	# ============================================================================
	# NIGHTLY K8S DEPLOYMENT
	# Build the operator, copy the nightly runtime images to the deploy registry,
	# then validate a baseline profile for each framework on the deploy-test runner.
	# ============================================================================

	vllm-copy-to-acr:
	name: vllm-runtime-nightly
	needs: [compute-release-mode, vllm-build]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-copy.yml
	with:
	target_tag_plain: ${{ format('{0}-nightly', needs.vllm-build.outputs.target_tag_plain) }}
	cuda_version: '["13.0"]'
	copy_timeout_minutes: 20
	secrets: inherit

	sglang-copy-to-acr:
	name: sglang-runtime-nightly
	needs: [compute-release-mode, sglang-build]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-copy.yml
	with:
	target_tag_plain: ${{ format('{0}-nightly', needs.sglang-build.outputs.target_tag_plain) }}
	cuda_version: '["13.0"]'
	copy_timeout_minutes: 20
	secrets: inherit

	trtllm-copy-to-acr:
	name: trtllm-runtime-nightly
	needs: [compute-release-mode, trtllm-build]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-copy.yml
	with:
	target_tag_plain: ${{ format('{0}-nightly', needs.trtllm-build.outputs.target_tag_plain) }}
	cuda_version: '["13.1"]'
	copy_timeout_minutes: 20
	secrets: inherit

	deploy-operator:
	needs: [compute-release-mode, operator-build, resolve-source-sha]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	runs-on: prod-deploy-tester-v1
	outputs:
	namespace: ${{ steps.setup.outputs.namespace }}
	vcluster_name: ${{ steps.setup.outputs.vcluster_name }}
	operator_tag: ${{ steps.setup.outputs.operator_tag }}
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	persist-credentials: false
	ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	- name: Setup vCluster and operator
	id: setup
	uses: $/.github/actions/setup-dynamo-operator
	with:
	# Sole writer of this vCluster name; every other caller recovers into it.
	reset_existing: 'true'
	vcluster_name: nightly-ci-${{ github.run_id }}
	vcluster_namespace: nightly-ci-${{ github.run_id }}
	registry: ${{ secrets.AZURE_ACR_HOSTNAME }}
	repository: ${{ vars.ACR_REPOSITORY }}
	operator_tag: ${{ needs.operator-build.outputs.operator_default_tag }}
	hf_token: ${{ secrets.HF_TOKEN }}
	dockerhub_username: ${{ secrets.DOCKERHUB_LOGIN_USER }}
	dockerhub_password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
	# Provision the shared model cache PV/PVC when the repo provides its
	# endpoint; empty vars -> no cache (workers download from HuggingFace).
	model_cache_server: ${{ vars.AZURE_MODEL_CACHE_SERVER }}
	model_cache_path: ${{ vars.AZURE_MODEL_CACHE_PATH }}

	deploy-test-vllm:
	name: vLLM Nightly Deploy Test
	needs: [compute-release-mode, deploy-operator, vllm-build, vllm-copy-to-acr]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-deploy-test.yml
	with:
	framework: vllm
	profiles: '["agg", "agg_router_kv_approx"]'
	image_tag: ${{ needs.vllm-copy-to-acr.outputs.image_tag }}
	namespace: ${{ needs.deploy-operator.outputs.namespace }}
	vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }}
	operator_tag: ${{ needs.deploy-operator.outputs.operator_tag }}
	secrets: inherit

	deploy-test-sglang:
	name: sglang Nightly Deploy Test
	needs: [compute-release-mode, deploy-operator, sglang-build, sglang-copy-to-acr]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-deploy-test.yml
	with:
	framework: sglang
	profiles: '["agg", "agg_logging"]'
	image_tag: ${{ needs.sglang-copy-to-acr.outputs.image_tag }}
	namespace: ${{ needs.deploy-operator.outputs.namespace }}
	vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }}
	operator_tag: ${{ needs.deploy-operator.outputs.operator_tag }}
	secrets: inherit

	deploy-test-trtllm:
	name: TensorRT-LLM Nightly Deploy Test
	needs: [compute-release-mode, deploy-operator, trtllm-build, trtllm-copy-to-acr]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-deploy-test.yml
	with:
	framework: trtllm
	profiles: '["agg"]'
	image_tag: ${{ needs.trtllm-copy-to-acr.outputs.image_tag }}
	namespace: ${{ needs.deploy-operator.outputs.namespace }}
	vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }}
	operator_tag: ${{ needs.deploy-operator.outputs.operator_tag }}
	secrets: inherit

	deploy-cleanup:
	name: Nightly Deploy Cleanup
	if: always()
	needs: [deploy-operator, deploy-test-vllm, deploy-test-sglang, deploy-test-trtllm]
	runs-on: prod-deploy-tester-v1
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	persist-credentials: false
	- name: Teardown vCluster
	if: needs.deploy-operator.outputs.namespace != '' && needs.deploy-operator.outputs.vcluster_name != ''
	uses: $/.github/actions/teardown-dynamo-operator
	with:
	vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }}
	vcluster_namespace: ${{ needs.deploy-operator.outputs.namespace }}

	# ============================================================================
	# EFA DEPLOY TEST
	# ============================================================================
	# The only jobs here that run against real EFA hardware. Everything else
	# validates the -efa images statically; these deploy a 1P1D stack on aws-dev-02
	# (p5 / H100, 32 EFA devices per node) -- vLLM first, then SGLang -- and prove
	# the prefill->decode KV transfer rode NIXL -> LIBFABRIC -> EFA instead of
	# silently falling back to UCX.
	#
	# The two framework tests share one vCluster and run one after the other rather
	# than together, to keep peak GPU use on the shared CI pool at one 1P1D stack.
	# SGLang is gated on the vLLM job having finished, not on it having passed
	# (`!cancelled()` rather than `success()`), so a vLLM regression does not also
	# delete the SGLang result.
	#
	# It cannot reuse shared-deploy-test.yml: that workflow hardcodes its runner,
	# timeout, ACR image reference and `-m framework_only` selector, and forwards no
	# kube context, and this needs the Teleport kubeconfig pointed at aws-dev-02.
	#
	# No tolerations or nodeSelector appear in tests/deploy/efa/disagg-efa.yaml --
	# the cluster's Kyverno deploy-test-pod-policy injects them for GPU pods in the
	# dynamo-deploy-test namespace, and what it injects differs per cluster.
	#
	# Split setup / test / cleanup into three jobs, matching deploy-operator ->
	# deploy-test-* -> deploy-cleanup above. Two properties of that shape matter and
	# neither survives collapsing it into one job:
	#
	# * Setup carries no timeout-minutes. setup-dynamo-operator's own waits
	# (vCluster 900s, CRD 1800s, operator readiness 1800s) can exceed any
	# reasonable job ceiling, so a ceiling spanning setup + test can expire
	# before the test even starts.
	# * Cleanup is a separate job with job-level `if: always()` over `needs:`, so
	# it fires however the upstream ended -- including job timeout, cancellation
	# and a dead runner, none of which a step-level `if: always()` survives.
	# That matters here because setup-dynamo-operator deliberately skips the
	# nscleanup/ttl label for dynamo-deploy-test, so a leaked vCluster holds p5
	# GPUs and EFA interfaces on a shared cluster with no automated reaper.
	efa-deploy-operator:
	name: EFA Deploy Operator
	needs: [compute-release-mode, operator-build, resolve-source-sha]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	runs-on: prod-deploy-tester-v1
	permissions:
	contents: read
	env:
	KUBE_CONTEXT: nv-prd-dgxc.teleport.sh-dynamo-aws-dev-02
	outputs:
	namespace: ${{ steps.setup.outputs.namespace }}
	vcluster_name: ${{ steps.setup.outputs.vcluster_name }}
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	persist-credentials: false
	ref: ${{ needs.resolve-source-sha.outputs.source_sha }}

	- name: Set kubectl context to aws-dev-02
	uses: ./.github/actions/set-kube-context

	# An empty vcluster_namespace makes the action resolve the shared
	# dynamo-deploy-test namespace, which is what aws-dev-02 expects.
	- name: Setup vCluster and operator
	id: setup
	uses: ./.github/actions/setup-dynamo-operator
	with:
	vcluster_name: nightly-efa-${{ github.run_id }}
	registry: ${{ secrets.AZURE_ACR_HOSTNAME }}
	repository: ${{ vars.ACR_REPOSITORY }}
	operator_tag: ${{ needs.operator-build.outputs.operator_default_tag }}
	hf_token: ${{ secrets.HF_TOKEN }}
	dockerhub_username: ${{ secrets.DOCKERHUB_LOGIN_USER }}
	dockerhub_password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}

	efa-deploy-test-vllm:
	name: vLLM EFA Deploy Test
	needs: [compute-release-mode, operator-build, resolve-source-sha, vllm-efa-build, efa-deploy-operator]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	runs-on: prod-deploy-tester-v1
	# Setup is already done in efa-deploy-operator, so this bounds the test alone.
	# The test carries @pytest.mark.timeout(1200) and its readiness wait is capped
	# at 900s below that, so pytest fails with a report first; this is the backstop
	# for a hang outside pytest. Safe to keep tight now that teardown is a separate
	# always() job rather than a step this would skip.
	timeout-minutes: 30
	permissions:
	contents: read
	env:
	KUBE_CONTEXT: nv-prd-dgxc.teleport.sh-dynamo-aws-dev-02
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	persist-credentials: false
	ref: ${{ needs.resolve-source-sha.outputs.source_sha }}

	- name: Set kubectl context to aws-dev-02
	uses: ./.github/actions/set-kube-context

	- name: Connect to vCluster
	id: connect-vcluster
	uses: ./.github/actions/connect-vcluster
	with:
	vcluster_name: ${{ needs.efa-deploy-operator.outputs.vcluster_name }}
	vcluster_namespace: ${{ needs.efa-deploy-operator.outputs.namespace }}

	- name: Run EFA deploy test
	uses: ./.github/actions/dynamo-deploy-test
	with:
	kubeconfig_base64: ${{ steps.connect-vcluster.outputs.kubeconfig_base64 }}
	namespace: default
	registry: ${{ secrets.AZURE_ACR_HOSTNAME }}
	operator_tag: ${{ needs.operator-build.outputs.operator_default_tag }}
	hf_token: ${{ secrets.HF_TOKEN }}
	framework: vllm
	test_name: vllm_efa
	test_file: tests/deploy/test_deploy_efa.py
	extra_pytest_args: -m framework_with_efa
	# Pulled straight from ECR, no copy-to-ACR hop and no imagePullSecret:
	# aws-dev-02 is an AWS cluster whose node instance role already grants
	# ECR read. Tag shape must stay in sync with calculate-target-tag in
	# shared-build-image.yml (cuda13 is the default, so it adds no suffix).
	image: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ needs.resolve-source-sha.outputs.source_sha }}-${{ needs.vllm-efa-build.outputs.target_tag_plain }}-nightly
	platform_arch: amd64

	efa-deploy-test-sglang:
	name: SGLang EFA Deploy Test
	needs: [compute-release-mode, operator-build, resolve-source-sha, sglang-efa-build, efa-deploy-operator, efa-deploy-test-vllm]
	# Runs after the vLLM test whatever that test concluded, so the two never hold
	# GPUs on the shared CI pool at the same time, but a vLLM failure still leaves
	# SGLang measured. Cancellation stops the lane; failure does not.
	#
	# `!cancelled()` replaces the default "every need succeeded" rule, so the two
	# needs this job genuinely cannot run without -- the vCluster and the image --
	# have to be spelled out again. Only efa-deploy-test-vllm is allowed to fail.
	if: ${{ !cancelled() && needs.compute-release-mode.outputs.run_tests == 'true' && needs.efa-deploy-operator.result == 'success' && needs.sglang-efa-build.result == 'success' }}
	runs-on: prod-deploy-tester-v1
	# Same reasoning as the vLLM job. Bring-up is dominated by the cold image
	# pull rather than by engine startup (measured on aws-dev-02: ~10 min to all
	# pods ready cold, ~1.5 min once the image is on the nodes), which the test's
	# own 900s readiness wait covers.
	timeout-minutes: 30
	permissions:
	contents: read
	env:
	KUBE_CONTEXT: nv-prd-dgxc.teleport.sh-dynamo-aws-dev-02
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	persist-credentials: false
	ref: ${{ needs.resolve-source-sha.outputs.source_sha }}

	- name: Set kubectl context to aws-dev-02
	uses: ./.github/actions/set-kube-context

	- name: Connect to vCluster
	id: connect-vcluster
	uses: ./.github/actions/connect-vcluster
	with:
	vcluster_name: ${{ needs.efa-deploy-operator.outputs.vcluster_name }}
	vcluster_namespace: ${{ needs.efa-deploy-operator.outputs.namespace }}

	# The vLLM stack normally tears itself down in the test's own fixture, but
	# that does not survive a job timeout or a lost runner, and
	# dynamo-deploy-test's fallback cleanup cannot help: its GRAPH_NAME reads
	# steps.deploy.outputs.graph_name, which the pytest step never sets, so it
	# always reports "already cleaned up". Without this, a stranded vLLM stack
	# would still hold GPUs while SGLang runs -- the one thing running the two
	# lanes sequentially is meant to prevent.
	- name: Delete any stranded vLLM EFA deployment
	shell: bash
	env:
	KUBECONFIG_B64: ${{ steps.connect-vcluster.outputs.kubeconfig_base64 }}
	PREFLIGHT_KUBECONFIG: ${{ github.workspace }}/.kubeconfig-preflight
	run: \|
	set -euo pipefail
	printf '%s' "${KUBECONFIG_B64}" \| base64 -d > "${PREFLIGHT_KUBECONFIG}"
	chmod 600 "${PREFLIGHT_KUBECONFIG}"
	export KUBECONFIG="${PREFLIGHT_KUBECONFIG}"
	kubectl delete dynamographdeployment vllm-disagg-efa \
	-n default --ignore-not-found --timeout=180s
	# Deletion returns once the DGD object is gone; the pods it owned can
	# outlive it, and those are what hold the GPUs and EFA devices.
	SEL="nvidia.com/dynamo-graph-deployment-name=vllm-disagg-efa"
	if ! kubectl wait --for=delete pod -l "${SEL}" -n default --timeout=180s; then
	# wait also exits non-zero when the selector matched nothing, which
	# is the normal path. Only pods that outlived the timeout matter, and
	# they are worth a warning rather than a failure: SGLang may still
	# schedule, and a hard failure here would lose that run entirely.
	REMAINING=$(kubectl get pod -l "${SEL}" -n default -o name \| wc -l \| tr -d ' ')
	[ "${REMAINING}" -eq 0 ] \|\| echo "::warning::${REMAINING} stranded vllm-disagg-efa pod(s) still hold GPUs/EFA devices"
	fi
	rm -f "${PREFLIGHT_KUBECONFIG}"

	- name: Run EFA deploy test
	uses: ./.github/actions/dynamo-deploy-test
	with:
	kubeconfig_base64: ${{ steps.connect-vcluster.outputs.kubeconfig_base64 }}
	namespace: default
	registry: ${{ secrets.AZURE_ACR_HOSTNAME }}
	operator_tag: ${{ needs.operator-build.outputs.operator_default_tag }}
	hf_token: ${{ secrets.HF_TOKEN }}
	framework: sglang
	test_name: sglang_efa
	test_file: tests/deploy/test_deploy_efa_sglang.py
	extra_pytest_args: -m framework_with_efa
	# Straight from ECR, as for vLLM above.
	image: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ needs.resolve-source-sha.outputs.source_sha }}-${{ needs.sglang-efa-build.outputs.target_tag_plain }}-nightly
	platform_arch: amd64

	efa-deploy-cleanup:
	name: EFA Deploy Cleanup
	if: always()
	needs: [efa-deploy-operator, efa-deploy-test-vllm, efa-deploy-test-sglang]
	runs-on: prod-deploy-tester-v1
	permissions:
	contents: read
	env:
	KUBE_CONTEXT: nv-prd-dgxc.teleport.sh-dynamo-aws-dev-02
	steps:
	# No ref: here, matching deploy-cleanup above. Teardown needs only the
	# vCluster name and namespace, both passed in from efa-deploy-operator's
	# outputs, so it does not depend on the commit this job checks out. Pinning
	# the resolved SHA also tripped CodeQL's untrusted-checkout and
	# cache-poisoning rules -- a false positive on a workflow with no
	# pull_request trigger, but not worth carrying for no behavioural gain.
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	persist-credentials: false

	- name: Set kubectl context to aws-dev-02
	uses: ./.github/actions/set-kube-context

	- name: Teardown vCluster
	if: needs.efa-deploy-operator.outputs.vcluster_name != '' && needs.efa-deploy-operator.outputs.namespace != ''
	uses: ./.github/actions/teardown-dynamo-operator
	with:
	vcluster_name: ${{ needs.efa-deploy-operator.outputs.vcluster_name }}
	vcluster_namespace: ${{ needs.efa-deploy-operator.outputs.namespace }}

	# ============================================================================
	# COMPLIANCE
	# ============================================================================
	# Compliance extract + verify now runs as an inline step INSIDE each build job
	# (inline_compliance: true above), on the same warm builder — a cache hit + small
	# scratch export instead of a cold full-image rebuild on the fresh builder. The
	# build jobs currently set archive_sources: false (archival temporarily
	# disabled — see above), so /sources.zip is not emitted; /legal + /sboms still
	# upload at the default 30-day retention. A compliance failure fails the build.

	# ============================================================================
	# TEST JOBS
	# ============================================================================

	vllm-test:
	name: vllm-runtime # This name overlaps with other vllm jobs to group them in the UI
	needs: [vllm-build, resolve-source-sha, compute-release-mode]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-test.yml
	with:
	dd_env: nightly
	dd_flaky_retry_enabled: 'false'
	test_suite_name: vllm
	framework: vllm # matches the framework passed to vllm-build
	test_type: Test
	amd_runner: prod-tester-amd-gpu-v2 # This runner is overridden for ARM platform
	target_tag_plain: ${{ needs.vllm-build.outputs.target_tag_plain }}
	cuda_version: '["13.0"]'
	platform: '["amd64", "arm64"]' # arm64 for CPU tests, single GPU tests are skipped
	enable_coverage: true
	run_cpu_only_tests: true
	cpu_only_test_markers: vllm and gpu_0
	# not sidecar: sidecar-marked tests need the dynamo-vllm-sidecar binary
	# installed (see pr.yaml's sidecar-vllm-test), which this plain
	# vllm-runtime image doesn't have.
	# TODO: this ad-hoc exclusion is duplicated across pr.yaml, post-merge-ci.yml,
	# and nightly-ci.yml's gpu_test_markers; consolidate into shared-test.yml
	# once there's a second marker that needs the same treatment.
	gpu_test_markers: vllm and gpu_1 and not sidecar
	gpu_test_timeout_minutes: 240
	# Profiled tests run in the parallel stage; unprofiled fall through to sequential.
	# 24 GiB admits all currently profiled vLLM tests (max is ~20.4 GiB) on a 48 GiB GPU.
	run_gpu_parallel_tests: true
	gpu_parallel_max_vram_gib: '24'
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	secrets: inherit

	vllm-multi-gpu-test:
	name: vllm-runtime # This name overlaps with other vllm jobs to group them in the UI
	needs: [vllm-build, resolve-source-sha, compute-release-mode]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-test.yml
	with:
	dd_env: nightly
	dd_flaky_retry_enabled: 'false'
	test_suite_name: vllm
	framework: vllm # matches the framework passed to vllm-build
	test_type: 2-GPU Test
	amd_runner: prod-tester-amd-gpu-2-v2
	target_tag_plain: ${{ needs.vllm-build.outputs.target_tag_plain }}
	cuda_version: '["13.0"]'
	platform: '["amd64"]' # No ARM GPUs available
	enable_coverage: true
	run_sanity_check: false
	gpu_test_markers: vllm and gpu_2 and not h100
	gpu_test_timeout_minutes: 45
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	secrets: inherit

	vllm-4-gpu-test:
	name: vllm-runtime # This name overlaps with other vllm jobs to group them in the UI
	needs: [vllm-build, resolve-source-sha, compute-release-mode]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-test.yml
	with:
	dd_env: nightly
	dd_flaky_retry_enabled: 'false'
	test_suite_name: vllm
	framework: vllm # matches the framework passed to vllm-build
	test_type: 4-GPU Test
	amd_runner: prod-tester-amd-gpu-4-v2
	target_tag_plain: ${{ needs.vllm-build.outputs.target_tag_plain }}
	cuda_version: '["13.0"]'
	platform: '["amd64"]' # No ARM GPUs available
	enable_coverage: true
	run_sanity_check: false
	# elastic_ep remains skipped because it needs at least 80 GiB per GPU.
	gpu_test_markers: vllm and gpu_4 and not h100
	gpu_test_timeout_minutes: 45
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	secrets: inherit

	vllm-h100-test:
	name: vllm-runtime # This name overlaps with other vllm jobs to group them in the UI
	needs: [vllm-build, resolve-source-sha, compute-release-mode]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-test.yml
	with:
	dd_env: nightly
	dd_flaky_retry_enabled: 'false'
	test_suite_name: vllm
	framework: vllm # matches the framework passed to vllm-build
	test_type: H100 Test
	# 2x 80 GiB H100 lane; runs vllm single- and dual-GPU tests, incl. h100-only cases.
	amd_runner: aws-dev-02-tester-amd-gpu-v2
	target_tag_plain: ${{ needs.vllm-build.outputs.target_tag_plain }}
	cuda_version: '["13.0"]'
	platform: '["amd64"]'
	enable_coverage: true
	run_sanity_check: false
	gpu_test_markers: vllm and (gpu_1 or gpu_2) and not sidecar
	gpu_test_timeout_minutes: 240
	# 80 GiB H100 admits more of the profiled pool per slot than the 24 GiB lanes.
	run_gpu_parallel_tests: true
	gpu_parallel_max_vram_gib: '80'
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	secrets: inherit

	sglang-test:
	name: sglang-runtime # This name overlaps with other sglang jobs to group them in the UI
	needs: [sglang-build, resolve-source-sha, compute-release-mode]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-test.yml
	with:
	dd_env: nightly
	dd_flaky_retry_enabled: 'false'
	test_suite_name: sglang
	framework: sglang # matches the framework passed to sglang-build
	test_type: Test
	amd_runner: prod-tester-amd-gpu-v2 # This runner is overridden for ARM platform
	target_tag_plain: ${{ needs.sglang-build.outputs.target_tag_plain }}
	cuda_version: '["13.0"]'
	platform: '["amd64", "arm64"]' # arm64 for CPU tests, single GPU tests are skipped
	enable_coverage: true
	run_cpu_only_tests: true
	cpu_only_test_markers: sglang and gpu_0
	# Exclude h100 tests: they need >24 GiB and run on the sglang-h100-test lane below.
	gpu_test_markers: sglang and gpu_1 and not h100 and not sidecar
	gpu_test_timeout_minutes: 240
	# Profiled tests run in the VRAM-aware GPU stage; unprofiled fall through to sequential.
	# Current single-GPU runners are 24 GiB, so this cap admits the profiled SGLang pool
	# while yielding one auto slot today. Larger runners will get more slots from the same markers.
	run_gpu_parallel_tests: true
	gpu_parallel_max_vram_gib: '24'
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	secrets: inherit

	sglang-multi-gpu-test:
	name: sglang-runtime # This name overlaps with other sglang jobs to group them in the UI
	needs: [sglang-build, resolve-source-sha, compute-release-mode]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-test.yml
	with:
	dd_env: nightly
	dd_flaky_retry_enabled: 'false'
	test_suite_name: sglang
	framework: sglang # matches the framework passed to sglang-build
	test_type: 2-GPU Test
	amd_runner: prod-tester-amd-gpu-2-v2
	target_tag_plain: ${{ needs.sglang-build.outputs.target_tag_plain }}
	cuda_version: '["13.0"]'
	platform: '["amd64"]' # No ARM GPUs available
	enable_coverage: true
	run_sanity_check: false
	gpu_test_markers: sglang and gpu_2 and not h100
	gpu_test_timeout_minutes: 45
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	secrets: inherit

	sglang-4-gpu-test:
	name: sglang-runtime # This name overlaps with other sglang jobs to group them in the UI
	needs: [sglang-build, resolve-source-sha, compute-release-mode]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-test.yml
	with:
	dd_env: nightly
	dd_flaky_retry_enabled: 'false'
	test_suite_name: sglang
	framework: sglang # matches the framework passed to sglang-build
	test_type: 4-GPU Test
	amd_runner: prod-tester-amd-gpu-4-v2
	target_tag_plain: ${{ needs.sglang-build.outputs.target_tag_plain }}
	cuda_version: '["13.0"]'
	platform: '["amd64"]' # No ARM GPUs available
	enable_coverage: true
	run_sanity_check: false
	gpu_test_markers: sglang and gpu_4 and not h100
	gpu_test_timeout_minutes: 45
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	secrets: inherit

	sglang-h100-test:
	name: sglang-runtime # This name overlaps with other sglang jobs to group them in the UI
	needs: [sglang-build, resolve-source-sha, compute-release-mode]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-test.yml
	with:
	dd_env: nightly
	dd_flaky_retry_enabled: 'false'
	test_suite_name: sglang
	framework: sglang # matches the framework passed to sglang-build
	test_type: H100 Test
	# 2x 80 GiB H100 lane; runs sglang single- and dual-GPU tests, incl. h100-only cases.
	amd_runner: aws-dev-02-tester-amd-gpu-v2
	target_tag_plain: ${{ needs.sglang-build.outputs.target_tag_plain }}
	cuda_version: '["13.0"]'
	platform: '["amd64"]'
	enable_coverage: true
	run_sanity_check: false
	gpu_test_markers: sglang and (gpu_1 or gpu_2) and not sidecar
	gpu_test_timeout_minutes: 240
	# 80 GiB H100 admits more of the profiled pool per slot than the 24 GiB lanes.
	run_gpu_parallel_tests: true
	gpu_parallel_max_vram_gib: '80'
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	secrets: inherit

	trtllm-test:
	name: trtllm-runtime # This name overlaps with other trtllm jobs to group them in the UI
	needs: [trtllm-build, resolve-source-sha, compute-release-mode]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-test.yml
	with:
	dd_env: nightly
	dd_flaky_retry_enabled: 'false'
	test_suite_name: trtllm
	framework: trtllm # matches the framework passed to trtllm-build
	test_type: Test
	amd_runner: prod-tester-amd-gpu-v2 # This runner is overridden for ARM platform
	target_tag_plain: ${{ needs.trtllm-build.outputs.target_tag_plain }}
	cuda_version: '["13.1"]'
	platform: '["amd64", "arm64"]' # arm64 for CPU tests, single GPU tests are skipped
	enable_coverage: true
	run_cpu_only_tests: true
	cpu_only_test_markers: trtllm and gpu_0
	gpu_test_markers: trtllm and gpu_1 and not sidecar
	gpu_test_timeout_minutes: 240
	# Profiled tests run in the VRAM-aware GPU stage; unprofiled fall through to sequential.
	# Current single-GPU runners are 24 GiB, so this cap admits the profiled TRT-LLM pool
	# while yielding one auto slot today. Larger runners will get more slots from the same markers.
	run_gpu_parallel_tests: true
	gpu_parallel_max_vram_gib: '24'
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	secrets: inherit

	trtllm-multi-gpu-test:
	name: trtllm-runtime # This name overlaps with other trtllm jobs to group them in the UI
	needs: [trtllm-build, resolve-source-sha, compute-release-mode]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-test.yml
	with:
	dd_env: nightly
	dd_flaky_retry_enabled: 'false'
	test_suite_name: trtllm
	framework: trtllm # matches the framework passed to trtllm-build
	test_type: 2-GPU Test
	amd_runner: prod-tester-amd-gpu-2-v2
	target_tag_plain: ${{ needs.trtllm-build.outputs.target_tag_plain }}
	cuda_version: '["13.1"]'
	platform: '["amd64"]' # No ARM GPUs available
	enable_coverage: true
	run_sanity_check: false
	gpu_test_markers: trtllm and gpu_2 and not h100
	gpu_test_timeout_minutes: 45
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	secrets: inherit

	trtllm-h100-test:
	name: trtllm-runtime # This name overlaps with other trtllm jobs to group them in the UI
	needs: [trtllm-build, resolve-source-sha, compute-release-mode]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/shared-test.yml
	with:
	dd_env: nightly
	dd_flaky_retry_enabled: 'false'
	test_suite_name: trtllm
	framework: trtllm # matches the framework passed to trtllm-build
	test_type: H100 Test
	# 2x 80 GiB H100 lane; runs trtllm single- and dual-GPU tests, incl. h100-only cases.
	amd_runner: aws-dev-02-tester-amd-gpu-v2
	target_tag_plain: ${{ needs.trtllm-build.outputs.target_tag_plain }}
	cuda_version: '["13.1"]'
	platform: '["amd64"]'
	enable_coverage: true
	run_sanity_check: false
	gpu_test_markers: trtllm and (gpu_1 or gpu_2) and not sidecar
	gpu_test_timeout_minutes: 240
	# 80 GiB H100 admits more of the profiled pool per slot than the 24 GiB lanes.
	run_gpu_parallel_tests: true
	gpu_parallel_max_vram_gib: '80'
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	secrets: inherit

	# ============================================================================
	# XPU TEST JOBS
	# ============================================================================

	vllm-test-xpu:
	name: vllm-xpu
	needs: [resolve-source-sha, compute-release-mode]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	uses: $/.github/workflows/xpu-ci.yaml
	with:
	framework: vllm
	pytest_markers: 'vllm'
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}

	# ============================================================================
	# DYNAMO RUNTIME PIPELINE
	# ============================================================================
	dynamo-pipeline:
	name: dynamo-runtime
	needs: [create-fresh-builder, compute-dev-version, resolve-source-sha]
	uses: $/.github/workflows/dynamo-pipeline.yml
	with:
	builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }}
	fresh_builder: true
	no_cache: true
	build_timeout_minutes: 120
	# TODO: widen beyond `pre_merge` — today it picks up tests
	# (e.g. fault_tolerance/deploy/*) that fail in this container-only
	# context. Matches the coverage of the old container-validation-dynamo
	# workflow.
	# `defaulted` is applied by the repository-root conftest.py to any test that
	# declared no lifecycle or hardware marker. It is routed here, not to the
	# sequential job: sequential runs the fault-tolerance suite in one process
	# and was OOM-killed (exit 137) once every unmarked test landed in it.
	# NOTE: this job is `-n 0` today (cpu_parallel_mode: 'none'), so `defaulted`
	# only picks the pod. Before enabling real xdist here, vet these tests for
	# parallel-safety or split them into their own job -- unlike `parallel`,
	# `defaulted` is not an author's claim that the test is parallel-safe.
	cpu_parallel_test_markers: 'pre_merge and (parallel or defaulted) and not (vllm or sglang or trtllm) and (gpu_0)'
	gpu_test_markers: 'pre_merge and none and gpu_1'
	dev_version_suffix: ${{ needs.compute-dev-version.outputs.dev_suffix }}
	source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
	image_tag_suffix: '-nightly'
	secrets: inherit

	# ============================================================================
	# RUST COVERAGE
	# ============================================================================
	rust-tests:
	name: rust-${{ matrix.dir == '.' && 'root' \|\| matrix.dir }}-coverage
	needs: [compute-release-mode]
	if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
	runs-on:
	group: Fastchecker
	strategy:
	fail-fast: false
	matrix:
	dir: ['.', 'lib/bindings/python', 'lib/bindings/kvbm']
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	lfs: true
	- name: Set up system dependencies
	run: \|
	# Install protoc for Rust build dependencies (NOTE: much faster than apt install)
	PB_REL="https://github.com/protocolbuffers/protobuf/releases"
	PROTOC_VER="30.2"
	PROTOC_ZIP="protoc-${PROTOC_VER}-linux-x86_64.zip"
	PROTOC_SHA256="327e9397c6fb3ea2a542513a3221334c6f76f7aa524a7d2561142b67b312a01f"
	curl --retry 5 --retry-delay 2 -fsSLO "$PB_REL/download/v${PROTOC_VER}/${PROTOC_ZIP}"
	echo "${PROTOC_SHA256} ${PROTOC_ZIP}" \| sha256sum -c -
	unzip "${PROTOC_ZIP}" -d $HOME/.local
	rm "${PROTOC_ZIP}"
	export PATH="$PATH:$HOME/.local/bin"
	protoc --version
	- name: Cache cargo artifacts
	uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
	with:
	path: \|
	~/.cargo/bin/
	~/.cargo/registry
	key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
	restore-keys: ${{ runner.os }}-cargo-
	- name: Set up Rust Toolchain Components
	run: rustup component add llvm-tools-preview
	- name: Install cargo-llvm-cov
	run: cargo-llvm-cov --version 2>/dev/null \|\| cargo install cargo-llvm-cov --locked
	# Have an explicit step to build tests first to separate time spent on build vs execution.
	- name: Compile Tests
	working-directory: ${{ matrix.dir }}
	run: cargo test --locked --no-run
	- name: Run Unit Tests with Coverage
	working-directory: ${{ matrix.dir }}
	# NOTE: --all-targets doesn't run doc tests.
	# cargo llvm-cov is a drop-in for cargo test; --no-report defers output
	# so we can generate multiple formats without re-running the tests.
	run: \|
	cargo llvm-cov --locked --all-targets --no-report
	cargo llvm-cov report --output-path coverage-rust.txt
	cargo llvm-cov report --lcov --output-path coverage-rust.lcov
	echo "Coverage summary:"
	grep "^TOTAL" coverage-rust.txt \|\| tail -3 coverage-rust.txt
	SAFE_DIR=$(echo "${{ matrix.dir }}" \| sed 's\|^\.$\|root\|' \| tr '/' '-')
	echo "RUST_COV_ARTIFACT_NAME=coverage-rust-${SAFE_DIR}-${{ github.run_id }}" >> $GITHUB_ENV
	- name: Upload Rust Coverage Data
	uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
	if: always()
	with:
	name: ${{ env.RUST_COV_ARTIFACT_NAME }}
	path: \|
	${{ matrix.dir }}/coverage-rust.txt
	${{ matrix.dir }}/coverage-rust.lcov
	retention-days: 7

	# ============================================================================
	# RELEASE (workflow_call into release.yml)
	# ============================================================================
	# commit_sha is the SHA `resolve-source-sha` picked (github.sha). release.yml
	# requires post-merge CI to have already pushed images to ECR for that SHA;
	# on schedule this is true because post-merge runs on every merge to main
	# and nightly fires later.
	release:
	name: Release Nightly
	needs:
	- resolve-source-sha
	- compute-release-mode
	- notify-slack-start
	- manual-release-approval
	- vllm-build
	- sglang-build
	- trtllm-build
	- dynamo-pipeline
	- operator-build
	- planner-build
	- frontend-build
	- vllm-efa-build
	- sglang-efa-build
	- trtllm-efa-build
	# !cancelled() lets the job evaluate the explicit gates below rather than
	# implicitly requiring every need to succeed (notify-slack-start is
	# best-effort). All three runtime builds and dynamo-pipeline are gated
	# strictly: release.yml copies their -nightly ECR images to NGC and extracts
	# wheels from the dynamo-pipeline image, so a failed build must not publish a
	# partial release. manual-release-approval is `skipped` on cron, `success` on dispatch.
	# The optional-image builds (operator, planner, frontend, EFA variants)
	# are ORDERING-ONLY needs (deliberately absent from the
	# gate below): release.yml copies them fail-soft, so a flake in any of
	# them must not block the runtime nightly — but the copy step has to run
	# after they finish or the crane probe would race the builds.
	if: ${{ !cancelled()
	&& needs.compute-release-mode.outputs.release == 'true'
	&& needs.vllm-build.result == 'success'
	&& needs.sglang-build.result == 'success'
	&& needs.trtllm-build.result == 'success'
	&& needs.dynamo-pipeline.result == 'success'
	&& (needs.manual-release-approval.result == 'success'
	\|\| needs.manual-release-approval.result == 'skipped') }}
	uses: $/.github/workflows/release.yml
	with:
	commit_sha: ${{ needs.resolve-source-sha.outputs.source_sha }}
	nightly: true
	skip_gitlab_pipeline: ${{ inputs.skip_gitlab_pipeline \|\| false }}
	slack_thread_ts: ${{ needs.notify-slack-start.outputs.thread_ts }}
	# THIS run's build results — release.yml's copy step skips either image
	# when its build here did not succeed, so a failed build can never
	# republish a previous run's image at an unchanged SHA. (A direct
	# workflow_dispatch of release.yml leaves these empty and stages from
	# existing -nightly ECR tags, matching the runtimes.)
	operator_build_result: ${{ needs.operator-build.result }}
	planner_build_result: ${{ needs.planner-build.result }}
	frontend_build_result: ${{ needs.frontend-build.result }}
	vllm_efa_build_result: ${{ needs.vllm-efa-build.result }}
	sglang_efa_build_result: ${{ needs.sglang-efa-build.result }}
	trtllm_efa_build_result: ${{ needs.trtllm-efa-build.result }}
	secrets: inherit

	# ============================================================================
	# COVERAGE REPORT
	# ============================================================================
	coverage-report:
	name: Generate Coverage Report
	runs-on: ubuntu-latest
	needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, rust-tests]
	if: always()
	permissions:
	contents: read
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

	- name: Set up Python
	uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
	with:
	python-version: '3.12'

	- name: Install Coverage Tools
	run: \|
	python -m pip install "coverage[toml]==7.13.1"
	python -m coverage --version
	echo "✅ Coverage tools installed"

	- name: Download All Python Coverage Artifacts
	uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
	with:
	pattern: coverage-python-*
	path: coverage-artifacts/
	merge-multiple: false

	- name: Download All Rust Coverage Artifacts
	uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
	continue-on-error: true
	with:
	pattern: coverage-rust-*
	path: coverage-rust-artifacts/
	merge-multiple: false

	- name: List Downloaded Artifacts
	run: \|
	echo "📦 Downloaded coverage artifacts:"
	echo "==== Directory structure (including hidden files) ===="
	ls -Ra coverage-artifacts/

	echo ""
	echo "==== Coverage files found ===="
	find coverage-artifacts/ -type f $ -name ".xml" -o -name ".coverage" $ \| sort
	echo ""
	echo "==== Specifically looking for .coverage files ===="
	find coverage-artifacts/ -name ".coverage" -type f
	echo ""

	- name: Merge All Coverage Data
	run: \|
	set -x
	echo "📊 Merging all test coverage..."

	# Expand $GITHUB_WORKSPACE in the config
	sed -i "s\|\$GITHUB_WORKSPACE\|${GITHUB_WORKSPACE}\|g" .coveragerc

	echo "Updated .coveragerc with path remapping:"
	cat .coveragerc

	# Find all .coverage files and copy them with unique names
	mkdir -p coverage-combined
	find coverage-artifacts/ -name ".coverage*" 2>/dev/null \| while read -r file; do
	unique_name=$(echo "$file" \| tr '/' '_' \| sed 's/coverage-artifacts_//')
	cp "$file" "coverage-combined/.coverage.${unique_name}"
	echo "Copied: $file -> coverage-combined/.coverage.${unique_name}"
	done

	# Check if we have any coverage files
	if ls coverage-combined/.coverage* 1> /dev/null 2>&1; then
	echo "✅ Found coverage files to merge"
	echo "Files to merge:"
	ls -lh coverage-combined/

	# Combine all coverage data from the workspace root so that the
	# relative canonical path "components/src/dynamo" in .coveragerc
	# resolves to $GITHUB_WORKSPACE/components/src/dynamo (where the
	# source actually lives) rather than to the non-existent
	# coverage-combined/components/src/dynamo subdirectory.
	echo "Running coverage combine with path remapping..."
	set +e # Don't exit on error
	COVERAGE_RCFILE=.coveragerc coverage combine --keep coverage-combined/.coverage* 2>&1 \| tee combine.log
	set -e

	if [ -f .coverage ]; then
	echo "✅ Combined .coverage file created"

	# Generate reports (continue even if some fail)
	echo "📊 Generating coverage reports..."
	set +e
	COVERAGE_RCFILE=.coveragerc coverage report --show-missing --data-file=.coverage 2>&1 \| tee coverage-report.txt
	TOTAL_COVERAGE=$(awk '/^TOTAL/ {print $NF}' coverage-report.txt \| tail -1)
	echo "TOTAL_COVERAGE=${TOTAL_COVERAGE:-0%}" >> $GITHUB_ENV
	COVERAGE_RCFILE=.coveragerc coverage html --data-file=.coverage -d coverage-html/ 2>&1 \|\| echo "HTML generation failed"
	COVERAGE_RCFILE=.coveragerc coverage xml --data-file=.coverage -o coverage-merged.xml 2>&1 \|\| echo "XML generation failed"
	set -e

	else
	echo "❌ Failed to create combined .coverage file"
	echo "No coverage data available" > coverage-report.txt
	echo "TOTAL_COVERAGE=0%" >> $GITHUB_ENV
	fi
	else
	echo "⚠️ No coverage data found"
	echo "No coverage data available" > coverage-report.txt
	echo "TOTAL_COVERAGE=0%" >> $GITHUB_ENV
	fi

	- name: Process Rust Coverage
	run: \|
	RUST_TOTAL_COVERAGE="N/A"
	echo "No Rust coverage data available yet." > coverage-rust-report.txt

	if find coverage-rust-artifacts/ -name "coverage-rust.txt" -type f 2>/dev/null \| grep -q .; then
	echo "📊 Processing Rust coverage reports..."
	# Concatenate full per-file coverage from every workspace dir
	> coverage-rust-report.txt
	find coverage-rust-artifacts/ -name "coverage-rust.txt" \| sort \| while read -r f; do
	DIR_NAME=$(basename $(dirname "$f"))
	echo "=== ${DIR_NAME} ===" >> coverage-rust-report.txt
	cat "$f" >> coverage-rust-report.txt
	echo "" >> coverage-rust-report.txt
	done

	# Aggregate line coverage across all crates by summing total/missed lines
	# from every TOTAL row. Columns on a `cargo llvm-cov report` TOTAL line:
	# $2 regions $3 missed_regions $4 region%
	# $5 funcs $6 missed_funcs $7 func%
	# $8 lines $9 missed_lines $10 line%
	RUST_TOTAL_COVERAGE=$(find coverage-rust-artifacts/ -name "coverage-rust.txt" -exec grep -h "^TOTAL" {} + \
	\| awk '{ total += $8; missed += $9 } END { if (total > 0) printf "%.2f%%", (total - missed) * 100 / total; else print "N/A" }')
	RUST_TOTAL_COVERAGE="${RUST_TOTAL_COVERAGE:-N/A}"
	echo "Rust line coverage (aggregated across $(find coverage-rust-artifacts/ -name "coverage-rust.txt" \| wc -l \| tr -d ' ') crates): ${RUST_TOTAL_COVERAGE}"
	else
	echo "ℹ️ No Rust coverage artifacts found"
	fi

	echo "RUST_TOTAL_COVERAGE=${RUST_TOTAL_COVERAGE}" >> $GITHUB_ENV

	- name: Create Coverage Summary
	run: \|
	DATE=$(date +"%Y-%m-%d %H:%M:%S UTC")
	cat > coverage-summary.md << EOF
	# 📊 Test Coverage Report

	Date: ${DATE}
	Run ID: ${{ github.run_id }}
	Workflow: ${{ github.workflow }}

	\| Language \| Total Coverage \|
	\|----------\|---------------\|
	\| Python \| ${TOTAL_COVERAGE} \|
	\| Rust \| ${RUST_TOTAL_COVERAGE} \|

	---

	## Python Coverage Details

	\`\`\`
	$(cat coverage-report.txt 2>/dev/null \|\| echo "No Python coverage data available")
	\`\`\`

	---

	## Rust Coverage Details

	\`\`\`
	$(cat coverage-rust-report.txt 2>/dev/null \|\| echo "No Rust coverage data available")
	\`\`\`

	---

	## 📁 Artifacts
	- Full HTML Report: Download \`coverage-reports-${{ github.run_id }}\` artifact
	- Python Coverage XML: \`coverage-merged.xml\`
	- Rust Coverage LCov: \`coverage-rust.lcov\` (per workspace dir)

	EOF

	echo "📄 Coverage summary generated"
	cat coverage-summary.md

	- name: Post to Workflow Summary
	run: cat coverage-summary.md >> $GITHUB_STEP_SUMMARY

	- name: Upload Coverage Reports
	uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
	if: always()
	with:
	name: coverage-reports-${{ github.run_id }}
	path: \|
	coverage-html/
	coverage-merged.xml
	coverage-report.txt
	coverage-rust-report.txt
	coverage-rust-artifacts/
	coverage-summary.md
	.coverage
	retention-days: 30

	# ============================================================================
	# CLEANUP
	# ============================================================================
	clean-k8s-builder:
	name: Clean K8s builder if exists
	runs-on: prod-default-small-v2
	if: always()
	needs: [vllm-build, sglang-build, trtllm-build, dynamo-pipeline, operator-build, planner-build, frontend-build, vllm-efa-build, sglang-efa-build, trtllm-efa-build, create-fresh-builder]
	permissions:
	contents: read
	steps:
	- name: Checkout repository
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	- name: Register K8s builder context (skip bootstrap)
	uses: $/.github/actions/bootstrap-buildkit
	continue-on-error: true
	with:
	builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }}
	buildkit_worker_addresses: ''
	namespace: ${{ vars.BUILDKIT_NAMESPACE }}
	skip_bootstrap: 'true'
	- name: Remove K8s builder
	shell: bash
	run: \|
	docker buildx rm ${{ env.BUILDER_NAME }} \|\| true

	############################## SLACK NOTIFICATION ##############################
	notify-slack:
	if: always()
	needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, deploy-cleanup, rust-tests]
	permissions:
	contents: read
	actions: read # grant the reusable notifier read access to list this run's jobs
	uses: $/.github/workflows/notify-slack.yml
	with:
	pipeline_name: Nightly
	runs_on: prod-default-v2
	secrets: inherit

Navigation Menu

Repository navigation

Nightly CI Pipeline #271

Workflow file

Nightly CI Pipeline #271

Uh oh!

Workflow file for this run