From d573f54a494c3f47f2e662ce6cde7aae00249810 Mon Sep 17 00:00:00 2001 From: Meihan-chen Date: Wed, 19 Aug 2026 11:22:36 +0800 Subject: [PATCH 1/5] Refactor NPU patch reconciliation Signed-off-by: Meihan-chen --- .buildkite/README-NPU-CI.md | 23 +- .buildkite/pipeline-npu.yaml | 11 +- .buildkite/scripts/update-npu-environment.sh | 291 +++++++++++-------- docker/Dockerfile.npu | 40 ++- docker/npu_patch/series.conf | 16 + 5 files changed, 230 insertions(+), 151 deletions(-) create mode 100644 docker/npu_patch/series.conf diff --git a/.buildkite/README-NPU-CI.md b/.buildkite/README-NPU-CI.md index 62fe7294c..f0b54b191 100644 --- a/.buildkite/README-NPU-CI.md +++ b/.buildkite/README-NPU-CI.md @@ -86,7 +86,22 @@ list in `SUITES`. ## Adding or removing a patch -Patches live in `docker/npu_patch/` and are registered in the `PATCH_CONFIGS` -dict in [`update-npu-environment.sh`](./scripts/update-npu-environment.sh). -Update both when adding or removing a patch. - +Patches are applied explicitly by [`docker/Dockerfile.npu`](../docker/Dockerfile.npu) +and reconciled at runtime in the order declared by +[`docker/npu_patch/series.conf`](../docker/npu_patch/series.conf). + +When adding a patch file, add the matching Dockerfile apply operation and the +`series.conf` entry in the same order. When deleting a patch, remove the +Dockerfile operation and `series.conf` entry before deleting the patch file; +the previous image retains the OLD bytes required for runtime revert. Rename +and reorder changes must likewise update both declarations. Ordinary patch +changes must not add patch-specific branches to the CI script. + +Patch-only changes reuse the default image and are reconciled when the test +container starts. Changes to repositories, dependencies, or the Docker build +environment require the contributor to select `image-build`. + +The initial rollout of this mechanism requires publishing a new default image +that contains `/opt/vime-npu/patch-state/docker/npu_patch/series.conf`, then +updating `DEFAULT_CI_IMAGE` in `npu_suites.py`. The legacy default image does not +contain the OLD patch state and cannot run this reconciler. diff --git a/.buildkite/pipeline-npu.yaml b/.buildkite/pipeline-npu.yaml index 1414faba9..c1fca23af 100644 --- a/.buildkite/pipeline-npu.yaml +++ b/.buildkite/pipeline-npu.yaml @@ -1,13 +1,8 @@ # Usage: buildkite-agent pipeline upload .buildkite/pipeline-npu.yaml # # This pipeline is for running NPU tests on PRs. -# Before execution, it checks: -# 1. Trigger type: PR trigger or scheduled trigger -# 2. Changed files: whether docker/Dockerfile.npu is modified (PR trigger only) -# Based on the check results: -# - Image build decision: scheduled trigger always builds; PR trigger builds only if Dockerfile.npu is modified -# - Test image selection: uses newly built image ($IMAGE_REGISTRY/$IMAGE_NAME:$VIME_IMAGE_TAG) when build is required, -# otherwise uses pre-built default image (quay.io/ascend/vime:0.3.0-a3-vllm0.22.1rc1) +# Scheduled builds always build an image. PR builds use the pre-built default +# image unless the contributor explicitly selects the image-build option. steps: - group: ":pipeline: Run NPU test" @@ -75,4 +70,4 @@ steps: SKIP_IMAGE_BUILD=true fi sed -e "s/__SKIP_IMAGE_BUILD__/$${SKIP_IMAGE_BUILD}/g" .buildkite/pipeline-npu-image.yaml | buildkite-agent pipeline upload - python .buildkite/npu_suites.py | buildkite-agent pipeline upload \ No newline at end of file + python .buildkite/npu_suites.py | buildkite-agent pipeline upload diff --git a/.buildkite/scripts/update-npu-environment.sh b/.buildkite/scripts/update-npu-environment.sh index 14abd11cf..b974cec08 100644 --- a/.buildkite/scripts/update-npu-environment.sh +++ b/.buildkite/scripts/update-npu-environment.sh @@ -1,165 +1,224 @@ #!/bin/bash -# Purpose: Updates NPU test environment to match PR changes -# - Saves and reverts all old patches before code update (for proper revert) -# - Updates VIME code to the specified commit -# - Applies all new patches to corresponding components -# - Sorts ASCEND_VISIBLE_DEVICES for consistent device ordering +# Purpose: Updates an NPU test container to match the requested VIME commit. +# - Reads the image's persisted OLD patch series and exact patch bytes +# - Updates VIME, then reconciles OLD -> NEW in declared series order +# - Installs the current VIME checkout and normalizes visible devices # Usage: Called by Buildkite pipeline during NPU test runs -set -e +set -e -o pipefail -VIME_DIR="/root/vime" +VIME_DIR="${VIME_DIR:-/root/vime}" +VIME_NPU_PATCH_STATE_DIR="${VIME_NPU_PATCH_STATE_DIR:-/opt/vime-npu/patch-state}" +PATCH_SERIES_RELATIVE_PATH="docker/npu_patch/series.conf" -declare -A PATCH_CONFIGS=( - ["vllm.patch"]="/vllm-workspace/vllm" - ["vllm-ascend.patch"]="/vllm-workspace/vllm-ascend" - ["megatron_comm.patch"]="/root/Megatron-LM" - ["megatron.patch"]="/root/Megatron-LM" - ["megatron-bridge.patch"]="/root/Megatron-Bridge" - ["mindspeed.patch"]="/root/MindSpeed" -) - -update_vime_code() { - echo "INFO: Updating VIME code..." - cd "$VIME_DIR" - - if [ -n "${BUILDKITE_COMMIT}" ]; then - echo "INFO: Fetching and checking out commit ${BUILDKITE_COMMIT}" - git fetch origin "${BUILDKITE_COMMIT}" - git checkout "${BUILDKITE_COMMIT}" - pip install -e . --no-deps --break-system-packages || pip install -e . --no-deps +sha256_stdin() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum | awk '{print $1}' else - echo "INFO: BUILDKITE_COMMIT not set, skipping code update" + shasum -a 256 | awk '{print $1}' fi } -sort_ascend_visible_devices() { - export ASCEND_VISIBLE_DEVICES="${ASCEND_VISIBLE_DEVICES:-$ASCEND_RT_VISIBLE_DEVICES}" - echo "Value: ${ASCEND_VISIBLE_DEVICES}" - if [ -n "${ASCEND_VISIBLE_DEVICES}" ]; then - SORTED_DEVICES=$(echo "${ASCEND_VISIBLE_DEVICES}" | tr ',' '\n' | sort -n | tr '\n' ',') - SORTED_DEVICES=${SORTED_DEVICES%,} - export ASCEND_VISIBLE_DEVICES=$SORTED_DEVICES - echo "Sorted ASCEND_VISIBLE_DEVICES: $ASCEND_VISIBLE_DEVICES" +sha256_file() { + local path="$1" + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$path" | awk '{print $1}' + else + shasum -a 256 "$path" | awk '{print $1}' fi } -get_patch_component() { - local patch_name="$1" - local config="${PATCH_CONFIGS[$patch_name]}" - echo "$config" -} +SERIES_ENTRIES=() -is_patch_applied() { - local component_dir="$1" - local patch_path="$2" +load_series() { + local series_file="$1" + local line target patch_file extra - if git -C "$component_dir" apply --reverse --check --whitespace=nowarn "$patch_path"; then - return 0 - else + if [ ! -f "$series_file" ]; then + echo "ERROR: Patch series not found: $series_file" >&2 return 1 fi -} -revert_patch() { - local component_dir="$1" - local patch_name="$2" - local old_patch_path="$3" - - if [ -f "$old_patch_path" ]; then - echo "INFO: Attempting to reverse-apply old patch from $old_patch_path" - if git -C "$component_dir" apply --reverse --whitespace=nowarn "$old_patch_path"; then - echo "INFO: Successfully reverted old $patch_name" - else - echo "WARNING: Failed to reverse-apply old patch $patch_name, skipping" + SERIES_ENTRIES=() + while IFS= read -r line || [ -n "$line" ]; do + if [[ "$line" =~ ^[[:space:]]*$ || "$line" =~ ^[[:space:]]*# ]]; then + continue fi - else - echo "INFO: Old patch $patch_name not found at $old_patch_path, skipping revert" - fi + + IFS='|' read -r target patch_file extra <<< "$line" + if [ -z "$target" ] || [ -z "$patch_file" ] || [ -n "$extra" ]; then + echo "ERROR: Invalid patch series entry: $line" >&2 + return 1 + fi + if [[ "$patch_file" = /* || "/$patch_file/" = *"/../"* ]]; then + echo "ERROR: Patch path must stay under the source root: $patch_file" >&2 + return 1 + fi + SERIES_ENTRIES+=("${target}|${patch_file}") + done < "$series_file" } -apply_patch() { - local component_dir="$1" +validate_series_entry() { + local target="$1" local patch_path="$2" - local patch_name="$3" - echo "INFO: Applying $patch_name to $component_dir" - if git -C "$component_dir" apply --whitespace=nowarn "$patch_path"; then - echo "INFO: Successfully applied $patch_name" - else - echo "ERROR: Failed to apply $patch_name to $component_dir" - exit 1 + if ! git -C "$target" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "ERROR: Patch target is not a Git worktree: $target" >&2 + return 1 + fi + if [ ! -f "$patch_path" ]; then + echo "ERROR: Patch file not found: $patch_path" >&2 + return 1 fi } -save_old_patches() { - local backup_dir="${VIME_DIR}/.old_patches" - - echo "INFO: Saving old patches to $backup_dir..." >&2 - mkdir -p "$backup_dir" - - for patch_name in "${!PATCH_CONFIGS[@]}"; do - local patch_path="${VIME_DIR}/docker/npu_patch/$patch_name" - if [ -f "$patch_path" ]; then - cp "$patch_path" "$backup_dir/$patch_name" - echo "INFO: Saved old $patch_name" >&2 - else - echo "WARNING: Patch file $patch_path not found, skipping backup" >&2 +series_digest() { + local series_file="$1" + local source_root="$2" + local entry target patch_file patch_path patch_sha + + load_series "$series_file" + for entry in "${SERIES_ENTRIES[@]}"; do + IFS='|' read -r target patch_file <<< "$entry" + patch_path="${source_root}/${patch_file}" + if [ ! -f "$patch_path" ]; then + echo "ERROR: Patch file not found: $patch_path" >&2 + return 1 fi done - echo "$backup_dir" + { + printf 'vime-npu-patch-series-v1\0' + for entry in "${SERIES_ENTRIES[@]}"; do + IFS='|' read -r target patch_file <<< "$entry" + patch_path="${source_root}/${patch_file}" + patch_sha=$(sha256_file "$patch_path") + printf '%s\0%s\0%s\0' "$target" "$patch_file" "$patch_sha" + done + } | sha256_stdin } -revert_all_patches() { - local old_patch_dir="$1" +apply_series() { + local series_file="$1" + local source_root="$2" + local entry target patch_file patch_path + + load_series "$series_file" + for entry in "${SERIES_ENTRIES[@]}"; do + IFS='|' read -r target patch_file <<< "$entry" + patch_path="${source_root}/${patch_file}" + validate_series_entry "$target" "$patch_path" + echo "INFO: Applying $patch_file to $target" + git -C "$target" apply --check --whitespace=nowarn "$patch_path" + git -C "$target" apply --whitespace=nowarn "$patch_path" + done +} - echo "INFO: Reverting all already applied patches..." +revert_series() { + local series_file="$1" + local source_root="$2" + local i entry target patch_file patch_path + + load_series "$series_file" + for ((i=${#SERIES_ENTRIES[@]}-1; i>=0; i--)); do + entry="${SERIES_ENTRIES[$i]}" + IFS='|' read -r target patch_file <<< "$entry" + patch_path="${source_root}/${patch_file}" + validate_series_entry "$target" "$patch_path" + echo "INFO: Reverting $patch_file from $target" + git -C "$target" apply --reverse --check --whitespace=nowarn "$patch_path" + git -C "$target" apply --reverse --whitespace=nowarn "$patch_path" + done +} - for patch_name in "${!PATCH_CONFIGS[@]}"; do - local component_dir=$(get_patch_component "$patch_name") - local old_patch_path="$old_patch_dir/$patch_name" +reconcile_series() { + local old_series="$1" + local old_root="$2" + local new_series="$3" + local new_root="$4" + local old_digest new_digest + + old_digest=$(series_digest "$old_series" "$old_root") + new_digest=$(series_digest "$new_series" "$new_root") + if [ "$old_digest" = "$new_digest" ]; then + echo "INFO: Patch series is unchanged" + return + fi - if is_patch_applied "$component_dir" "$old_patch_path"; then - echo "INFO: $patch_name is currently applied, reverting..." - revert_patch "$component_dir" "$patch_name" "$old_patch_path" - else - echo "INFO: $patch_name is not currently applied, nothing to revert" - fi - done + revert_series "$old_series" "$old_root" + apply_series "$new_series" "$new_root" } -apply_all_patches() { - echo "INFO: Applying all patches..." +update_vime_code() { + echo "INFO: Updating VIME code..." - for patch_name in "${!PATCH_CONFIGS[@]}"; do - local component_dir=$(get_patch_component "$patch_name") - local patch_path="${VIME_DIR}/docker/npu_patch/$patch_name" + if [ -n "${BUILDKITE_COMMIT:-}" ]; then + echo "INFO: Fetching and checking out commit ${BUILDKITE_COMMIT}" + git -C "$VIME_DIR" fetch origin "${BUILDKITE_COMMIT}" + git -C "$VIME_DIR" checkout "${BUILDKITE_COMMIT}" + else + echo "INFO: BUILDKITE_COMMIT not set, skipping code update" + fi +} - if [ -f "$patch_path" ]; then - apply_patch "$component_dir" "$patch_path" "$patch_name" - else - echo "WARNING: Patch file $patch_path not found, skipping" - fi - done +install_vime_code() { + pip install -e "$VIME_DIR" --no-deps --break-system-packages || pip install -e "$VIME_DIR" --no-deps +} + +sort_ascend_visible_devices() { + export ASCEND_VISIBLE_DEVICES="${ASCEND_VISIBLE_DEVICES:-${ASCEND_RT_VISIBLE_DEVICES:-}}" + echo "Value: ${ASCEND_VISIBLE_DEVICES}" + if [ -n "${ASCEND_VISIBLE_DEVICES}" ]; then + SORTED_DEVICES=$(echo "${ASCEND_VISIBLE_DEVICES}" | tr ',' '\n' | sort -n | tr '\n' ',') + SORTED_DEVICES=${SORTED_DEVICES%,} + export ASCEND_VISIBLE_DEVICES=$SORTED_DEVICES + echo "Sorted ASCEND_VISIBLE_DEVICES: $ASCEND_VISIBLE_DEVICES" + fi } main() { + local old_series="${VIME_NPU_PATCH_STATE_DIR}/${PATCH_SERIES_RELATIVE_PATH}" + local new_series="${VIME_DIR}/${PATCH_SERIES_RELATIVE_PATH}" + echo "=== Step 1: Sort ASCEND_VISIBLE_DEVICES ===" sort_ascend_visible_devices - echo "=== Step 2: Save all old patches before code update ===" - local old_patch_dir=$(save_old_patches) - - echo "=== Step 3: Update VIME code ===" + echo "=== Step 2: Update VIME code ===" update_vime_code - echo "=== Step 4: Revert all already applied patches ===" - revert_all_patches "$old_patch_dir" + if [ ! -f "$old_series" ]; then + echo "ERROR: The selected image does not contain NPU patch state: $old_series" >&2 + echo "ERROR: Build a patch-state-enabled NPU image before running this commit." >&2 + return 1 + fi + + echo "=== Step 3: Reconcile image patches with current VIME patches ===" + reconcile_series "$old_series" "$VIME_NPU_PATCH_STATE_DIR" "$new_series" "$VIME_DIR" - echo "=== Step 5: Apply all patches ===" - apply_all_patches + echo "=== Step 4: Install current VIME code ===" + install_vime_code echo "INFO: NPU environment update completed successfully" } -main \ No newline at end of file +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + case "${1:-}" in + series-digest) + if [ "$#" -ne 3 ]; then + echo "Usage: $0 series-digest SERIES_FILE SOURCE_ROOT" >&2 + exit 2 + fi + series_digest "$2" "$3" + exit + ;; + reconcile) + if [ "$#" -ne 5 ]; then + echo "Usage: $0 reconcile OLD_SERIES OLD_ROOT NEW_SERIES NEW_ROOT" >&2 + exit 2 + fi + reconcile_series "$2" "$3" "$4" "$5" + exit + ;; + esac +fi + +main diff --git a/docker/Dockerfile.npu b/docker/Dockerfile.npu index 22e1fb18b..fc39dbb80 100644 --- a/docker/Dockerfile.npu +++ b/docker/Dockerfile.npu @@ -29,9 +29,12 @@ ENV DEBIAN_FRONTEND=noninteractive \ HYDRA_FULL_ERROR=1 \ PYTHONPATH=/root/Megatron-Bridge/src:/root/Megatron-LM:/root/vime -# Patches copied from build context; always apply from /tmp/npu_patch so local/PR -# builds use the same files as the build context (not the later git-cloned vime tree). -COPY docker/npu_patch /tmp/npu_patch +# PATCH MAINTENANCE: when adding, deleting, renaming, or reordering a patch, +# update the explicit apply operations below and docker/npu_patch/series.conf in +# the same change. Ordinary patch changes must not require CI script changes. +# Apply from the persisted state directory so runtime CI retains the exact OLD +# bytes and order needed to reconcile this image with a later VIME checkout. +COPY docker/npu_patch /opt/vime-npu/patch-state/docker/npu_patch RUN git config --global http.sslVerify false @@ -51,14 +54,14 @@ RUN if [ -n "$APTMIRROR" ];then sed -i "s@^\(deb.*\)https\?://[a-z0-9.-]*\.ubunt # vllm and vllm-ascend are installed editable (-e) in the base image; apply the # NPU colocate patches directly to their source trees. RUN git -C /vllm-workspace/vllm apply --check --whitespace=nowarn \ - /tmp/npu_patch/vllm.patch && \ + /opt/vime-npu/patch-state/docker/npu_patch/vllm.patch && \ git -C /vllm-workspace/vllm apply --whitespace=nowarn \ - /tmp/npu_patch/vllm.patch + /opt/vime-npu/patch-state/docker/npu_patch/vllm.patch RUN git -C /vllm-workspace/vllm-ascend apply --check --whitespace=nowarn \ - /tmp/npu_patch/vllm-ascend.patch && \ + /opt/vime-npu/patch-state/docker/npu_patch/vllm-ascend.patch && \ git -C /vllm-workspace/vllm-ascend apply --whitespace=nowarn \ - /tmp/npu_patch/vllm-ascend.patch + /opt/vime-npu/patch-state/docker/npu_patch/vllm-ascend.patch # Protect the serving stack while installing Vime dependencies. RUN python3 -c 'import importlib.metadata as m; names = ["numpy", "ray", "torch", "torch-npu", "torchvision", "transformers", "triton-ascend", "vllm", "vllm-ascend"]; open("/tmp/vime-npu-constraints.txt", "w").write("\n".join(f"{name}=={m.version(name)}" for name in names) + "\n")' @@ -81,24 +84,15 @@ RUN git clone https://gitcode.com/Ascend/MindSpeed.git /root/MindSpeed && \ RUN git clone https://github.com/ISEEKYAN/mbridge.git /root/mbridge && \ git -C /root/mbridge checkout "${MBRIDGE_COMMIT}" -# Apply NPU training-stack patches from build context (/tmp/npu_patch). -RUN for patch_file in /tmp/npu_patch/*.patch; do \ - base="$(basename "${patch_file}")"; \ - case "${base}" in \ - vllm-ascend.patch|vllm.patch) continue ;; \ - esac; \ - if [[ "$(tail -c 1 "${patch_file}" | od -An -t x1 | tr -d ' \n')" != "0a" ]]; then \ - printf '\n' >> "${patch_file}"; \ - fi; \ - done && \ +# Apply NPU training-stack patches from the persisted build-context snapshot. +RUN git -C /root/Megatron-LM apply --whitespace=nowarn \ + /opt/vime-npu/patch-state/docker/npu_patch/megatron_comm.patch && \ git -C /root/Megatron-LM apply --whitespace=nowarn \ - /tmp/npu_patch/megatron_comm.patch && \ - git -C /root/Megatron-LM apply --whitespace=nowarn \ - /tmp/npu_patch/megatron.patch && \ + /opt/vime-npu/patch-state/docker/npu_patch/megatron.patch && \ git -C /root/Megatron-Bridge apply --whitespace=nowarn \ - /tmp/npu_patch/megatron-bridge.patch && \ + /opt/vime-npu/patch-state/docker/npu_patch/megatron-bridge.patch && \ git -C /root/MindSpeed apply --whitespace=nowarn \ - /tmp/npu_patch/mindspeed.patch + /opt/vime-npu/patch-state/docker/npu_patch/mindspeed.patch # Megatron-Bridge is used directly from PYTHONPATH. Installing its package # metadata would pull CUDA-only dependencies into the Ascend environment. @@ -130,4 +124,4 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ WORKDIR /root/vime ENTRYPOINT [] -CMD ["/bin/bash"] \ No newline at end of file +CMD ["/bin/bash"] diff --git a/docker/npu_patch/series.conf b/docker/npu_patch/series.conf new file mode 100644 index 000000000..3fb20b032 --- /dev/null +++ b/docker/npu_patch/series.conf @@ -0,0 +1,16 @@ +# Ordered NPU patch series used by the runtime CI reconciler. +# Format: target_worktree|patch_file (apply top-to-bottom, revert bottom-to-top). +# +# PATCH MAINTENANCE: +# - Add: add the patch file and its explicit apply operation to +# docker/Dockerfile.npu, then add this entry in the same application order. +# - Delete: remove the explicit Dockerfile operation and this entry, then delete +# the patch file. The previous image keeps the OLD bytes needed for revert. +# - Rename/reorder: update both Dockerfile.npu and this file in the same change. +# Ordinary patch changes must not add patch-specific logic to the CI script. +/vllm-workspace/vllm|docker/npu_patch/vllm.patch +/vllm-workspace/vllm-ascend|docker/npu_patch/vllm-ascend.patch +/root/Megatron-LM|docker/npu_patch/megatron_comm.patch +/root/Megatron-LM|docker/npu_patch/megatron.patch +/root/Megatron-Bridge|docker/npu_patch/megatron-bridge.patch +/root/MindSpeed|docker/npu_patch/mindspeed.patch From 44773c08571d028822790f4aba90ebd5d9f6e790 Mon Sep 17 00:00:00 2001 From: Meihan-chen Date: Wed, 19 Aug 2026 14:09:29 +0800 Subject: [PATCH 2/5] Refine NPU patch reconciliation Signed-off-by: Meihan-chen --- .buildkite/README-NPU-CI.md | 18 +++-- .buildkite/pipeline-npu.yaml | 4 +- .buildkite/scripts/update-npu-environment.sh | 78 ++++++++++++-------- docker/Dockerfile.npu | 27 +++---- docker/npu_patch/series.conf | 19 ++--- 5 files changed, 82 insertions(+), 64 deletions(-) diff --git a/.buildkite/README-NPU-CI.md b/.buildkite/README-NPU-CI.md index f0b54b191..6cd2e7a21 100644 --- a/.buildkite/README-NPU-CI.md +++ b/.buildkite/README-NPU-CI.md @@ -89,19 +89,23 @@ list in `SUITES`. Patches are applied explicitly by [`docker/Dockerfile.npu`](../docker/Dockerfile.npu) and reconciled at runtime in the order declared by [`docker/npu_patch/series.conf`](../docker/npu_patch/series.conf). +Each entry declares `target_worktree|image_patch|source_patch`: the image keeps +the flat `image_patch` under `/opt/npu_patch`, while runtime CI reads +`source_patch` relative to the current VIME checkout. -When adding a patch file, add the matching Dockerfile apply operation and the -`series.conf` entry in the same order. When deleting a patch, remove the +When adding a patch file, add the matching Dockerfile COPY/apply operations and +the `series.conf` entry in the same order. When deleting a patch, remove the Dockerfile operation and `series.conf` entry before deleting the patch file; the previous image retains the OLD bytes required for runtime revert. Rename and reorder changes must likewise update both declarations. Ordinary patch changes must not add patch-specific branches to the CI script. Patch-only changes reuse the default image and are reconciled when the test -container starts. Changes to repositories, dependencies, or the Docker build -environment require the contributor to select `image-build`. +container starts. Changes that add, remove, or upgrade external repositories, +modify installed dependencies, or otherwise change the Docker image must be +validated with `image-build`. The initial rollout of this mechanism requires publishing a new default image -that contains `/opt/vime-npu/patch-state/docker/npu_patch/series.conf`, then -updating `DEFAULT_CI_IMAGE` in `npu_suites.py`. The legacy default image does not -contain the OLD patch state and cannot run this reconciler. +that contains `/opt/npu_patch/series.conf`, then updating +`DEFAULT_CI_IMAGE` in `npu_suites.py`. The legacy default image contains patch +bytes under `/tmp/npu_patch` but not the OLD series required by this reconciler. diff --git a/.buildkite/pipeline-npu.yaml b/.buildkite/pipeline-npu.yaml index c1fca23af..244e860c6 100644 --- a/.buildkite/pipeline-npu.yaml +++ b/.buildkite/pipeline-npu.yaml @@ -1,8 +1,8 @@ # Usage: buildkite-agent pipeline upload .buildkite/pipeline-npu.yaml # # This pipeline is for running NPU tests on PRs. -# Scheduled builds always build an image. PR builds use the pre-built default -# image unless the contributor explicitly selects the image-build option. +# Scheduled builds always build a fresh image. PR builds reuse DEFAULT_CI_IMAGE +# unless image-build is selected; patch-only changes are reconciled at runtime. steps: - group: ":pipeline: Run NPU test" diff --git a/.buildkite/scripts/update-npu-environment.sh b/.buildkite/scripts/update-npu-environment.sh index b974cec08..37e617f7e 100644 --- a/.buildkite/scripts/update-npu-environment.sh +++ b/.buildkite/scripts/update-npu-environment.sh @@ -7,7 +7,8 @@ set -e -o pipefail VIME_DIR="${VIME_DIR:-/root/vime}" -VIME_NPU_PATCH_STATE_DIR="${VIME_NPU_PATCH_STATE_DIR:-/opt/vime-npu/patch-state}" +VIME_NPU_PATCH_STATE_DIR="${VIME_NPU_PATCH_STATE_DIR:-/opt/npu_patch}" +VIME_NPU_PATCH_SOURCE_ROOT="${VIME_NPU_PATCH_SOURCE_ROOT:-${VIME_DIR}}" PATCH_SERIES_RELATIVE_PATH="docker/npu_patch/series.conf" sha256_stdin() { @@ -31,7 +32,7 @@ SERIES_ENTRIES=() load_series() { local series_file="$1" - local line target patch_file extra + local line target image_patch source_patch extra if [ ! -f "$series_file" ]; then echo "ERROR: Patch series not found: $series_file" >&2 @@ -44,16 +45,20 @@ load_series() { continue fi - IFS='|' read -r target patch_file extra <<< "$line" - if [ -z "$target" ] || [ -z "$patch_file" ] || [ -n "$extra" ]; then + IFS='|' read -r target image_patch source_patch extra <<< "$line" + if [ -z "$target" ] || [ -z "$image_patch" ] || [ -z "$source_patch" ] || [ -n "$extra" ]; then echo "ERROR: Invalid patch series entry: $line" >&2 return 1 fi - if [[ "$patch_file" = /* || "/$patch_file/" = *"/../"* ]]; then - echo "ERROR: Patch path must stay under the source root: $patch_file" >&2 + if [[ "$image_patch" = */* ]]; then + echo "ERROR: Image patch must be a flat file name: $image_patch" >&2 return 1 fi - SERIES_ENTRIES+=("${target}|${patch_file}") + if [[ "$source_patch" = /* || "/$source_patch/" = *"/../"* ]]; then + echo "ERROR: Source patch must stay under the VIME root: $source_patch" >&2 + return 1 + fi + SERIES_ENTRIES+=("${target}|${image_patch}|${source_patch}") done < "$series_file" } @@ -73,13 +78,19 @@ validate_series_entry() { series_digest() { local series_file="$1" - local source_root="$2" - local entry target patch_file patch_path patch_sha + local patch_root="$2" + local path_kind="$3" + local entry target image_patch source_patch patch_ref patch_path patch_sha load_series "$series_file" for entry in "${SERIES_ENTRIES[@]}"; do - IFS='|' read -r target patch_file <<< "$entry" - patch_path="${source_root}/${patch_file}" + IFS='|' read -r target image_patch source_patch <<< "$entry" + case "$path_kind" in + image) patch_ref="$image_patch" ;; + source) patch_ref="$source_patch" ;; + *) echo "ERROR: Unknown patch path kind: $path_kind" >&2; return 1 ;; + esac + patch_path="${patch_root}/${patch_ref}" if [ ! -f "$patch_path" ]; then echo "ERROR: Patch file not found: $patch_path" >&2 return 1 @@ -89,10 +100,15 @@ series_digest() { { printf 'vime-npu-patch-series-v1\0' for entry in "${SERIES_ENTRIES[@]}"; do - IFS='|' read -r target patch_file <<< "$entry" - patch_path="${source_root}/${patch_file}" + IFS='|' read -r target image_patch source_patch <<< "$entry" + if [ "$path_kind" = "image" ]; then + patch_ref="$image_patch" + else + patch_ref="$source_patch" + fi + patch_path="${patch_root}/${patch_ref}" patch_sha=$(sha256_file "$patch_path") - printf '%s\0%s\0%s\0' "$target" "$patch_file" "$patch_sha" + printf '%s\0%s\0%s\0%s\0' "$target" "$image_patch" "$source_patch" "$patch_sha" done } | sha256_stdin } @@ -100,14 +116,14 @@ series_digest() { apply_series() { local series_file="$1" local source_root="$2" - local entry target patch_file patch_path + local entry target image_patch source_patch patch_path load_series "$series_file" for entry in "${SERIES_ENTRIES[@]}"; do - IFS='|' read -r target patch_file <<< "$entry" - patch_path="${source_root}/${patch_file}" + IFS='|' read -r target image_patch source_patch <<< "$entry" + patch_path="${source_root}/${source_patch}" validate_series_entry "$target" "$patch_path" - echo "INFO: Applying $patch_file to $target" + echo "INFO: Applying $source_patch to $target" git -C "$target" apply --check --whitespace=nowarn "$patch_path" git -C "$target" apply --whitespace=nowarn "$patch_path" done @@ -115,16 +131,16 @@ apply_series() { revert_series() { local series_file="$1" - local source_root="$2" - local i entry target patch_file patch_path + local image_root="$2" + local i entry target image_patch source_patch patch_path load_series "$series_file" for ((i=${#SERIES_ENTRIES[@]}-1; i>=0; i--)); do entry="${SERIES_ENTRIES[$i]}" - IFS='|' read -r target patch_file <<< "$entry" - patch_path="${source_root}/${patch_file}" + IFS='|' read -r target image_patch source_patch <<< "$entry" + patch_path="${image_root}/${image_patch}" validate_series_entry "$target" "$patch_path" - echo "INFO: Reverting $patch_file from $target" + echo "INFO: Reverting $image_patch from $target" git -C "$target" apply --reverse --check --whitespace=nowarn "$patch_path" git -C "$target" apply --reverse --whitespace=nowarn "$patch_path" done @@ -137,8 +153,8 @@ reconcile_series() { local new_root="$4" local old_digest new_digest - old_digest=$(series_digest "$old_series" "$old_root") - new_digest=$(series_digest "$new_series" "$new_root") + old_digest=$(series_digest "$old_series" "$old_root" image) + new_digest=$(series_digest "$new_series" "$new_root" source) if [ "$old_digest" = "$new_digest" ]; then echo "INFO: Patch series is unchanged" return @@ -176,8 +192,8 @@ sort_ascend_visible_devices() { } main() { - local old_series="${VIME_NPU_PATCH_STATE_DIR}/${PATCH_SERIES_RELATIVE_PATH}" - local new_series="${VIME_DIR}/${PATCH_SERIES_RELATIVE_PATH}" + local old_series="${VIME_NPU_PATCH_STATE_DIR}/series.conf" + local new_series="${VIME_NPU_PATCH_SOURCE_ROOT}/${PATCH_SERIES_RELATIVE_PATH}" echo "=== Step 1: Sort ASCEND_VISIBLE_DEVICES ===" sort_ascend_visible_devices @@ -192,7 +208,7 @@ main() { fi echo "=== Step 3: Reconcile image patches with current VIME patches ===" - reconcile_series "$old_series" "$VIME_NPU_PATCH_STATE_DIR" "$new_series" "$VIME_DIR" + reconcile_series "$old_series" "$VIME_NPU_PATCH_STATE_DIR" "$new_series" "$VIME_NPU_PATCH_SOURCE_ROOT" echo "=== Step 4: Install current VIME code ===" install_vime_code @@ -203,11 +219,11 @@ main() { if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then case "${1:-}" in series-digest) - if [ "$#" -ne 3 ]; then - echo "Usage: $0 series-digest SERIES_FILE SOURCE_ROOT" >&2 + if [ "$#" -ne 4 ]; then + echo "Usage: $0 series-digest SERIES_FILE PATCH_ROOT image|source" >&2 exit 2 fi - series_digest "$2" "$3" + series_digest "$2" "$3" "$4" exit ;; reconcile) diff --git a/docker/Dockerfile.npu b/docker/Dockerfile.npu index fc39dbb80..beeb41c52 100644 --- a/docker/Dockerfile.npu +++ b/docker/Dockerfile.npu @@ -29,12 +29,9 @@ ENV DEBIAN_FRONTEND=noninteractive \ HYDRA_FULL_ERROR=1 \ PYTHONPATH=/root/Megatron-Bridge/src:/root/Megatron-LM:/root/vime -# PATCH MAINTENANCE: when adding, deleting, renaming, or reordering a patch, -# update the explicit apply operations below and docker/npu_patch/series.conf in -# the same change. Ordinary patch changes must not require CI script changes. -# Apply from the persisted state directory so runtime CI retains the exact OLD -# bytes and order needed to reconcile this image with a later VIME checkout. -COPY docker/npu_patch /opt/vime-npu/patch-state/docker/npu_patch +# PATCH MAINTENANCE: keep patch COPY/apply operations and +# docker/npu_patch/series.conf synchronized. +COPY docker/npu_patch /opt/npu_patch RUN git config --global http.sslVerify false @@ -54,14 +51,14 @@ RUN if [ -n "$APTMIRROR" ];then sed -i "s@^\(deb.*\)https\?://[a-z0-9.-]*\.ubunt # vllm and vllm-ascend are installed editable (-e) in the base image; apply the # NPU colocate patches directly to their source trees. RUN git -C /vllm-workspace/vllm apply --check --whitespace=nowarn \ - /opt/vime-npu/patch-state/docker/npu_patch/vllm.patch && \ + /opt/npu_patch/vllm.patch && \ git -C /vllm-workspace/vllm apply --whitespace=nowarn \ - /opt/vime-npu/patch-state/docker/npu_patch/vllm.patch + /opt/npu_patch/vllm.patch RUN git -C /vllm-workspace/vllm-ascend apply --check --whitespace=nowarn \ - /opt/vime-npu/patch-state/docker/npu_patch/vllm-ascend.patch && \ + /opt/npu_patch/vllm-ascend.patch && \ git -C /vllm-workspace/vllm-ascend apply --whitespace=nowarn \ - /opt/vime-npu/patch-state/docker/npu_patch/vllm-ascend.patch + /opt/npu_patch/vllm-ascend.patch # Protect the serving stack while installing Vime dependencies. RUN python3 -c 'import importlib.metadata as m; names = ["numpy", "ray", "torch", "torch-npu", "torchvision", "transformers", "triton-ascend", "vllm", "vllm-ascend"]; open("/tmp/vime-npu-constraints.txt", "w").write("\n".join(f"{name}=={m.version(name)}" for name in names) + "\n")' @@ -84,15 +81,15 @@ RUN git clone https://gitcode.com/Ascend/MindSpeed.git /root/MindSpeed && \ RUN git clone https://github.com/ISEEKYAN/mbridge.git /root/mbridge && \ git -C /root/mbridge checkout "${MBRIDGE_COMMIT}" -# Apply NPU training-stack patches from the persisted build-context snapshot. +# Apply NPU training-stack patches from the build-context snapshot. RUN git -C /root/Megatron-LM apply --whitespace=nowarn \ - /opt/vime-npu/patch-state/docker/npu_patch/megatron_comm.patch && \ + /opt/npu_patch/megatron_comm.patch && \ git -C /root/Megatron-LM apply --whitespace=nowarn \ - /opt/vime-npu/patch-state/docker/npu_patch/megatron.patch && \ + /opt/npu_patch/megatron.patch && \ git -C /root/Megatron-Bridge apply --whitespace=nowarn \ - /opt/vime-npu/patch-state/docker/npu_patch/megatron-bridge.patch && \ + /opt/npu_patch/megatron-bridge.patch && \ git -C /root/MindSpeed apply --whitespace=nowarn \ - /opt/vime-npu/patch-state/docker/npu_patch/mindspeed.patch + /opt/npu_patch/mindspeed.patch # Megatron-Bridge is used directly from PYTHONPATH. Installing its package # metadata would pull CUDA-only dependencies into the Ascend environment. diff --git a/docker/npu_patch/series.conf b/docker/npu_patch/series.conf index 3fb20b032..2729cfe34 100644 --- a/docker/npu_patch/series.conf +++ b/docker/npu_patch/series.conf @@ -1,16 +1,17 @@ # Ordered NPU patch series used by the runtime CI reconciler. -# Format: target_worktree|patch_file (apply top-to-bottom, revert bottom-to-top). +# Format: target_worktree|image_patch|source_patch. +# Apply top-to-bottom from source_patch; revert bottom-to-top from image_patch. # # PATCH MAINTENANCE: -# - Add: add the patch file and its explicit apply operation to -# docker/Dockerfile.npu, then add this entry in the same application order. +# - Add: add the required COPY/apply operations to docker/Dockerfile.npu, then +# add this entry in the same application order. # - Delete: remove the explicit Dockerfile operation and this entry, then delete # the patch file. The previous image keeps the OLD bytes needed for revert. # - Rename/reorder: update both Dockerfile.npu and this file in the same change. # Ordinary patch changes must not add patch-specific logic to the CI script. -/vllm-workspace/vllm|docker/npu_patch/vllm.patch -/vllm-workspace/vllm-ascend|docker/npu_patch/vllm-ascend.patch -/root/Megatron-LM|docker/npu_patch/megatron_comm.patch -/root/Megatron-LM|docker/npu_patch/megatron.patch -/root/Megatron-Bridge|docker/npu_patch/megatron-bridge.patch -/root/MindSpeed|docker/npu_patch/mindspeed.patch +/vllm-workspace/vllm|vllm.patch|docker/npu_patch/vllm.patch +/vllm-workspace/vllm-ascend|vllm-ascend.patch|docker/npu_patch/vllm-ascend.patch +/root/Megatron-LM|megatron_comm.patch|docker/npu_patch/megatron_comm.patch +/root/Megatron-LM|megatron.patch|docker/npu_patch/megatron.patch +/root/Megatron-Bridge|megatron-bridge.patch|docker/npu_patch/megatron-bridge.patch +/root/MindSpeed|mindspeed.patch|docker/npu_patch/mindspeed.patch From 58aa1031fe5a6e5f00ce3d89b8803696da98e04e Mon Sep 17 00:00:00 2001 From: Meihan-chen Date: Wed, 19 Aug 2026 14:35:51 +0800 Subject: [PATCH 3/5] Refine NPU patch reconciliation review fixes Signed-off-by: Meihan-chen --- .buildkite/npu_suites.py | 2 +- .buildkite/scripts/update-npu-environment.sh | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.buildkite/npu_suites.py b/.buildkite/npu_suites.py index 3eac0ebf4..2715d0ca4 100644 --- a/.buildkite/npu_suites.py +++ b/.buildkite/npu_suites.py @@ -19,7 +19,7 @@ import sys NPU_QUEUE = "ascend-a3" -DEFAULT_CI_IMAGE = "quay.io/ascend/vime:0.3.0-a3-vllm0.22.1rc1" +DEFAULT_CI_IMAGE = "quay.io/ascend/vime:vime-latest" IMAGE_REGISTRY = "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry" IMAGE_NAME = "vime-ci-npu" VIME_IMAGE_TAG = os.environ.get("BUILDKITE_COMMIT", "latest") diff --git a/.buildkite/scripts/update-npu-environment.sh b/.buildkite/scripts/update-npu-environment.sh index 37e617f7e..761f2406d 100644 --- a/.buildkite/scripts/update-npu-environment.sh +++ b/.buildkite/scripts/update-npu-environment.sh @@ -34,12 +34,12 @@ load_series() { local series_file="$1" local line target image_patch source_patch extra + SERIES_ENTRIES=() if [ ! -f "$series_file" ]; then echo "ERROR: Patch series not found: $series_file" >&2 return 1 fi - SERIES_ENTRIES=() while IFS= read -r line || [ -n "$line" ]; do if [[ "$line" =~ ^[[:space:]]*$ || "$line" =~ ^[[:space:]]*# ]]; then continue @@ -124,7 +124,6 @@ apply_series() { patch_path="${source_root}/${source_patch}" validate_series_entry "$target" "$patch_path" echo "INFO: Applying $source_patch to $target" - git -C "$target" apply --check --whitespace=nowarn "$patch_path" git -C "$target" apply --whitespace=nowarn "$patch_path" done } @@ -141,7 +140,6 @@ revert_series() { patch_path="${image_root}/${image_patch}" validate_series_entry "$target" "$patch_path" echo "INFO: Reverting $image_patch from $target" - git -C "$target" apply --reverse --check --whitespace=nowarn "$patch_path" git -C "$target" apply --reverse --whitespace=nowarn "$patch_path" done } From 80832d84388ed3a944d3df2b6a621d79c315ac64 Mon Sep 17 00:00:00 2001 From: Meihan-chen Date: Tue, 21 Jul 2026 16:17:01 +0800 Subject: [PATCH 4/5] vlm Signed-off-by: Meihan-chen --- .buildkite/npu_suites.py | 2 + docker/npu_patch/mindspeed.patch | 21 +++-- tests/test_qwen3_vl_8B_npu.py | 139 +++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 6 deletions(-) create mode 100644 tests/test_qwen3_vl_8B_npu.py diff --git a/.buildkite/npu_suites.py b/.buildkite/npu_suites.py index 2715d0ca4..275afd125 100644 --- a/.buildkite/npu_suites.py +++ b/.buildkite/npu_suites.py @@ -30,6 +30,7 @@ "smk": [ ("test_qwen3_4B_npu.py", "npu-8", "", {}), ("test_qwen3_30B_A3B_npu.py", "npu-16", "", {}), + ("test_qwen3_vl_8B_npu.py", "npu-8", "", {}), ], "nightly": [], } @@ -89,6 +90,7 @@ def npu_step(suite: str, test_name: str, resource_class: str, extra_args: str, e 'if [ -n "${BUILDKITE_COMMIT}" ]; then', " source /workspace/build/buildkite/.buildkite/scripts/update-npu-environment.sh", "fi", + "export HF_HUB_OFFLINE=0", f"python tests/{test_name}{' ' + extra_args if extra_args else ''}", ] ) diff --git a/docker/npu_patch/mindspeed.patch b/docker/npu_patch/mindspeed.patch index a252107b1..0585af488 100644 --- a/docker/npu_patch/mindspeed.patch +++ b/docker/npu_patch/mindspeed.patch @@ -15,14 +15,23 @@ index bb007b44..98708a5b 100644 self.permute_idx_device = None input_chunk_idxs = torch.arange( diff --git a/mindspeed/core/megatron_basic/arguments_basic.py b/mindspeed/core/megatron_basic/arguments_basic.py -index 8ea25b9f..04ad9663 100644 +index 8ea25b9f..7853bce4 100644 --- a/mindspeed/core/megatron_basic/arguments_basic.py +++ b/mindspeed/core/megatron_basic/arguments_basic.py -@@ -113,3 +113,26 @@ def transformer_config_init_wrapper(fn): +@@ -113,3 +113,35 @@ def transformer_config_init_wrapper(fn): fn(self, *args, **known_config) return wrapper + ++ ++def transformer_config_getattr(self, name): ++ """Resolve MindSpeed extension fields for configs created before patching.""" ++ full_args = vars(get_full_args()) ++ if name in full_args: ++ return full_args[name] ++ raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") ++ ++ +def transformer_config_init_subclass(cls, **kwargs): + mutable_types = (list, dict, set, bytearray) + unknown_config = {} @@ -45,25 +54,25 @@ index 8ea25b9f..04ad9663 100644 + else: + value = value + setattr(cls, key, value) -\ No newline at end of file diff --git a/mindspeed/features_manager/megatron_basic/megatron_basic.py b/mindspeed/features_manager/megatron_basic/megatron_basic.py -index 355913e1..18714baf 100644 +index 355913e1..41678a03 100644 --- a/mindspeed/features_manager/megatron_basic/megatron_basic.py +++ b/mindspeed/features_manager/megatron_basic/megatron_basic.py -@@ -43,8 +43,11 @@ class MegatronBasicFeature(MindSpeedFeature): +@@ -43,8 +43,13 @@ class MegatronBasicFeature(MindSpeedFeature): def register_mcore_basic_patches(self, pm, args): # configuration patches - from mindspeed.core.megatron_basic.arguments_basic import transformer_config_init_wrapper, transformer_config_post_init_wrapper + from mindspeed.core.megatron_basic.arguments_basic import (transformer_config_init_wrapper, ++ transformer_config_getattr, + transformer_config_post_init_wrapper, + transformer_config_init_subclass) pm.register_patch("megatron.core.transformer.transformer_config.TransformerConfig.__init__", transformer_config_init_wrapper) + pm.register_patch("megatron.core.transformer.transformer_config.TransformerConfig.__init_subclass__", classmethod(transformer_config_init_subclass)) ++ pm.register_patch("megatron.core.transformer.transformer_config.TransformerConfig.__getattr__", transformer_config_getattr, create_dummy=True) pm.register_patch("megatron.core.transformer.transformer_config.MLATransformerConfig.__init__", transformer_config_init_wrapper) pm.register_patch("megatron.core.transformer.transformer_config.TransformerConfig.__post_init__", transformer_config_post_init_wrapper) - diff --git a/mindspeed/patch_utils.py b/mindspeed/patch_utils.py index a489d58c..d9328555 100644 --- a/mindspeed/patch_utils.py diff --git a/tests/test_qwen3_vl_8B_npu.py b/tests/test_qwen3_vl_8B_npu.py new file mode 100644 index 000000000..ff0577753 --- /dev/null +++ b/tests/test_qwen3_vl_8B_npu.py @@ -0,0 +1,139 @@ +import os +import shlex + +import vime.utils.external_utils.command_utils as U + + +# Single-turn Qwen3-VL GRPO on geo3k (mirrors examples/geo3k_vlm/run_geo3k_vlm_npu.sh). +# Qwen3-VL-8B maps to the qwen3-8B megatron config; the vision tower is handled by bridge. +MODEL_NAME = "Qwen3-VL-8B-Instruct" +MODEL_TYPE = "qwen3-8B" +TEST_ROOT = os.environ.get("HF_HOME") or "/root" +MODEL_DIR = f"{TEST_ROOT}/models/{MODEL_NAME}" +DATASET_DIR = f"{TEST_ROOT}/datasets/geo3k_imgurl" + + +def prepare(): + models_dir = shlex.quote(f"{TEST_ROOT}/models") + datasets_dir = shlex.quote(f"{TEST_ROOT}/datasets") + model_dir = shlex.quote(MODEL_DIR) + dataset_dir = shlex.quote(DATASET_DIR) + + U.exec_command(f"mkdir -p {models_dir} {datasets_dir}") + U.exec_command(f"hf download Qwen/{MODEL_NAME} --local-dir {model_dir}") + U.exec_command(f"hf download --repo-type dataset chenhegu/geo3k_imgurl --local-dir {dataset_dir}") + + +def execute(): + model_dir = shlex.quote(MODEL_DIR) + prompt_data = shlex.quote(f"{DATASET_DIR}/train.parquet") + + checkpoint_args = ( + f"--hf-checkpoint {model_dir} " f"--load {model_dir} " "--megatron-to-hf-mode bridge " "--no-load-optim " + ) + + rollout_args = ( + f"--prompt-data {prompt_data} " + "--input-key problem " + "--label-key answer " + '--multimodal-keys \'{"image": "images"}\' ' + "--apply-chat-template " + "--rollout-shuffle " + "--rm-type math " + "--num-rollout 2 " + "--rollout-batch-size 4 " + "--n-samples-per-prompt 4 " + "--rollout-max-response-len 4096 " + "--rollout-temperature 1 " + "--global-batch-size 16 " + ) + + parallel_args = ( + "--tensor-model-parallel-size 4 " + "--sequence-parallel " + "--pipeline-model-parallel-size 1 " + "--context-parallel-size 1 " + "--expert-model-parallel-size 1 " + "--expert-tensor-parallel-size 1 " + "--recompute-granularity full " + "--recompute-method uniform " + "--recompute-num-layers 1 " + "--use-dynamic-batch-size " + "--max-tokens-per-gpu 4096 " + ) + + grpo_args = ( + "--advantage-estimator grpo " + "--kl-loss-coef 0.00 " + "--kl-loss-type low_var_kl " + "--kl-coef 0.00 " + "--entropy-coef 0.00 " + "--eps-clip 0.2 " + "--eps-clip-high 0.28 " + ) + + optimizer_args = ( + "--optimizer adam " + "--lr 1e-6 " + "--lr-decay-style constant " + "--weight-decay 0.1 " + "--adam-beta1 0.9 " + "--adam-beta2 0.98 " + ) + + vllm_args = ( + "--rollout-num-gpus-per-engine 1 " + "--vllm-gpu-memory-utilization 0.8 " + "--vllm-max-model-len 16384 " + "--vllm-generation-config auto " + "--vllm-logprobs-mode processed_logprobs " + ) + + model_args = ( + "--attention-dropout 0.0 " + "--hidden-dropout 0.0 " + "--accumulate-allreduce-grads-in-fp32 " + "--attention-softmax-in-fp32 " + "--attention-backend flash " + "--use-flash-attn " + "--no-gradient-accumulation-fusion " + ) + + runtime_args = ( + "--train-backend megatron " + "--actor-num-nodes 1 " + "--actor-num-gpus-per-node 8 " + "--rollout-num-gpus 8 " + "--colocate " + "--ci-test " + ) + + train_args = ( + checkpoint_args + + rollout_args + + parallel_args + + grpo_args + + optimizer_args + + vllm_args + + model_args + + runtime_args + ) + # qwen3-8B.sh builds MODEL_ARGS with --rotary-base ${MODEL_ARGS_ROTARY_BASE}; Qwen3-VL needs 5e6. + os.environ["MODEL_ARGS_ROTARY_BASE"] = "5000000" + U.execute_train( + train_args=train_args, + num_gpus_per_node=8, + megatron_model_type=MODEL_TYPE, + extra_env_vars={}, + ) + + +def main(): + prepare() + for proxy_var in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY"): + os.environ.pop(proxy_var, None) + execute() + + +if __name__ == "__main__": + main() From 34b2d9ebc153ed5bc8cc9ee1338010062f0ac851 Mon Sep 17 00:00:00 2001 From: Meihan-chen Date: Thu, 20 Aug 2026 17:57:14 +0800 Subject: [PATCH 5/5] arg vime commit Signed-off-by: Meihan-chen --- docker/Dockerfile.npu | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile.npu b/docker/Dockerfile.npu index beeb41c52..17d9d6782 100644 --- a/docker/Dockerfile.npu +++ b/docker/Dockerfile.npu @@ -15,7 +15,8 @@ ARG SOC_VERSION="ascend910_9391" ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" ARG APTMIRROR="" -ENV DEBIAN_FRONTEND=noninteractive \ +ENV SOC_VERSION=$SOC_VERSION \ + DEBIAN_FRONTEND=noninteractive \ PIP_NO_CACHE_DIR=1 \ ASCEND_TOOLKIT_HOME=/usr/local/Ascend/ascend-toolkit/latest \ ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp \ @@ -63,10 +64,6 @@ RUN git -C /vllm-workspace/vllm-ascend apply --check --whitespace=nowarn \ # Protect the serving stack while installing Vime dependencies. RUN python3 -c 'import importlib.metadata as m; names = ["numpy", "ray", "torch", "torch-npu", "torchvision", "transformers", "triton-ascend", "vllm", "vllm-ascend"]; open("/tmp/vime-npu-constraints.txt", "w").write("\n".join(f"{name}=={m.version(name)}" for name in names) + "\n")' -# Vime: colocate NPU code lives on the ascend branch (not main/npu). -RUN git clone --depth 1 --branch ascend \ - https://github.com/vllm-project/vime.git /root/vime - # Training source dependencies. RUN git clone https://github.com/NVIDIA/Megatron-LM.git /root/Megatron-LM && \ git -C /root/Megatron-LM checkout "${MEGATRON_COMMIT}" @@ -98,6 +95,15 @@ RUN pip install --no-build-isolation "nvidia-modelopt[torch]>=0.37.0" && \ pip install --no-deps --no-build-isolation -e /root/Megatron-LM && \ pip install --no-deps --no-build-isolation -e /root/MindSpeed +# Defaults to the ascend branch for local builds. Release workflows should +# pass an immutable commit SHA for reproducible images. +ARG VIME_COMMIT=ascend +RUN test -n "${VIME_COMMIT}" && \ + git init /root/vime && \ + git -C /root/vime remote add origin https://github.com/vllm-project/vime.git && \ + git -C /root/vime fetch --depth 1 origin "${VIME_COMMIT}" && \ + git -C /root/vime checkout --detach FETCH_HEAD + # ring_flash_attn is a CUDA extension and is not used on Ascend. RUN pip install \ --constraint /tmp/vime-npu-constraints.txt \