From 5ed8679e8f901c8a6fcc855d60b80ff53f0c2c84 Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Mon, 27 Jul 2026 07:52:58 -0700 Subject: [PATCH 1/2] docker: remove the base image's apt cuDNN so the pip one is what loads The apt copy shadows the pip copy in ldconfig, and the loader interleaves the two: libcudnn_heuristic.so.9 comes from apt while libcudnn_graph.so.9 comes from pip. transformer_engine then fails either on import with an undefined symbol or inside fused attention with CUDNN_STATUS_BAD_PARAM, which is why moving the pip pin alone changed nothing. Verified on radixark/miles:pr-1795@sha256:792974ed with 8 DotProductAttention forward+backward configs (d=64/128 x MHA/GQA x sbhd/bshd): apt present + pip 9.16.0.29 -> torch refuses to init cuDNN apt present + pip 9.22.0.52 -> 8/8 fail apt removed + pip 9.19.0.56 -> 8/8 fail (the version floor is real) apt removed + pip 9.22.0.52 -> 8/8 pass --- docker/Dockerfile | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 945c7fe3e3..256ad30bcc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -162,6 +162,23 @@ RUN pip install multi-storage-client --no-deps COPY requirements.txt /tmp/requirements.txt RUN rm -rf /usr/lib/python3/dist-packages/jwt /usr/lib/python3/dist-packages/PyJWT* && pip install -r /tmp/requirements.txt +# The base image also carries cuDNN as apt packages, in /usr/lib/ where +# ldconfig finds it ahead of the pip copy. The two then get interleaved at dlopen +# time -- libcudnn_heuristic.so.9 resolves to the apt build while libcudnn_graph.so.9 +# resolves to the pip one -- and transformer_engine dies either on import +# (undefined symbol _ZTVN5cudnn7backend12OperationSetE) or inside fused attention. +# The apt copy is held, so removing it needs --allow-change-held-packages; without +# that flag apt skips it silently and the pip pin below stays inert. +RUN if [ "${ENABLE_CUDA_13}" = "1" ]; then CU=13; else CU=12; fi; \ + apt-get remove -y --purge --allow-change-held-packages \ + libcudnn9-cuda-${CU} libcudnn9-dev-cuda-${CU} libcudnn9-headers-cuda-${CU} \ + && rm -rf /var/lib/apt/lists/* + +# transformer_engine 2.17's fused attention emits cudnn-frontend's +# Reshape_attributes::set_reshape_mode, i.e. the backend attribute +# CUDNN_ATTR_OPERATION_RESHAPE_MODE, which cuDNN only understands from 9.22.0. +# torch 2.11 ships 9.19.0.56, so this has to raise it rather than just get out of +# the way: with the apt copy gone, 9.19.0.56 still fails and 9.22.0.52 passes. RUN if [ "${ENABLE_CUDA_13}" = "1" ]; then \ pip install nvidia-cudnn-cu13==9.22.0.52; \ else \ From b735234d7a225ff887b2efc6ad9d74fb636272cc Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Mon, 27 Jul 2026 09:13:09 -0700 Subject: [PATCH 2/2] docker: drop the version-history comments --- docker/Dockerfile | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 256ad30bcc..3feff79beb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -162,23 +162,14 @@ RUN pip install multi-storage-client --no-deps COPY requirements.txt /tmp/requirements.txt RUN rm -rf /usr/lib/python3/dist-packages/jwt /usr/lib/python3/dist-packages/PyJWT* && pip install -r /tmp/requirements.txt -# The base image also carries cuDNN as apt packages, in /usr/lib/ where -# ldconfig finds it ahead of the pip copy. The two then get interleaved at dlopen -# time -- libcudnn_heuristic.so.9 resolves to the apt build while libcudnn_graph.so.9 -# resolves to the pip one -- and transformer_engine dies either on import -# (undefined symbol _ZTVN5cudnn7backend12OperationSetE) or inside fused attention. -# The apt copy is held, so removing it needs --allow-change-held-packages; without -# that flag apt skips it silently and the pip pin below stays inert. +# The apt copy shadows the pip one in ldconfig and the loader interleaves the two, +# so transformer_engine gets a mixed set of libcudnn sub-libraries. The packages are +# held, hence --allow-change-held-packages. RUN if [ "${ENABLE_CUDA_13}" = "1" ]; then CU=13; else CU=12; fi; \ apt-get remove -y --purge --allow-change-held-packages \ libcudnn9-cuda-${CU} libcudnn9-dev-cuda-${CU} libcudnn9-headers-cuda-${CU} \ && rm -rf /var/lib/apt/lists/* -# transformer_engine 2.17's fused attention emits cudnn-frontend's -# Reshape_attributes::set_reshape_mode, i.e. the backend attribute -# CUDNN_ATTR_OPERATION_RESHAPE_MODE, which cuDNN only understands from 9.22.0. -# torch 2.11 ships 9.19.0.56, so this has to raise it rather than just get out of -# the way: with the apt copy gone, 9.19.0.56 still fails and 9.22.0.52 passes. RUN if [ "${ENABLE_CUDA_13}" = "1" ]; then \ pip install nvidia-cudnn-cu13==9.22.0.52; \ else \