[release/2.8] Update CentOS dockerfile for CentOS Stream 9

.ci/docker/build.sh

@@ -56,10 +56,14 @@ elif [[ "$image" == *-noble* ]]; then
   UBUNTU_VERSION=24.04
 elif [[ "$image" == *ubuntu* ]]; then
   extract_version_from_image_name ubuntu UBUNTU_VERSION
+elif [[ "$image" == *centos* ]]; then
+  extract_version_from_image_name centos CENTOS_VERSION
 fi
 
 if [ -n "${UBUNTU_VERSION}" ]; then
   OS="ubuntu"
+elif [ -n "${CENTOS_VERSION}" ]; then
+  OS="centos"
 else
   echo "Unable to derive operating system base..."
   exit 1

.ci/docker/centos-rocm/Dockerfile

@@ -1,8 +1,7 @@
 ARG CENTOS_VERSION
 
-FROM centos:${CENTOS_VERSION}
+FROM quay.io/centos/centos:stream${CENTOS_VERSION}
 
-ARG CENTOS_VERSION
 
 # Set AMD gpu targets to build for
 ARG PYTORCH_ROCM_ARCH
@@ -14,19 +13,21 @@ ENV PYTORCH_ROCM_ARCH ${PYTORCH_ROCM_ARCH}
 COPY ./common/install_base.sh install_base.sh
 RUN bash ./install_base.sh && rm install_base.sh
 
+#Install langpack
+RUN yum install -y glibc-langpack-en
+
 # Update CentOS git version
 RUN yum -y remove git
 RUN yum -y remove git-*
-RUN yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo-1.9-1.x86_64.rpm && \
-    sed -i 's/packages.endpoint/packages.endpointdev/' /etc/yum.repos.d/endpoint.repo
 RUN yum install -y git
 
 # Install devtoolset
-ARG DEVTOOLSET_VERSION
-COPY ./common/install_devtoolset.sh install_devtoolset.sh
-RUN bash ./install_devtoolset.sh && rm install_devtoolset.sh
+RUN dnf install -y rpmdevtools
 ENV BASH_ENV "/etc/profile"
 
+# Install ninja
+RUN dnf --enablerepo=crb install -y ninja-build
+
 # (optional) Install non-default glibc version
 ARG GLIBC_VERSION
 COPY ./common/install_glibc.sh install_glibc.sh
@@ -47,7 +48,7 @@ COPY ./common/install_conda.sh install_conda.sh
 COPY ./common/common_utils.sh common_utils.sh
 RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt
 
-# (optional) Install vision packages like OpenCV
+# (optional) Install vision packages like OpenCV and ffmpeg
 ARG VISION
 COPY ./common/install_vision.sh ./common/cache_vision_models.sh ./common/common_utils.sh ./
 RUN if [ -n "${VISION}" ]; then bash ./install_vision.sh; fi

.ci/docker/common/install_base.sh

@@ -1,95 +1,105 @@
-#!/bin/bash
+	#!/bin/bash
 
-set -ex
+	set -ex
 
-install_ubuntu() {
-  # NVIDIA dockers for RC releases use tag names like `11.0-cudnn9-devel-ubuntu18.04-rc`,
-  # for this case we will set UBUNTU_VERSION to `18.04-rc` so that the Dockerfile could
-  # find the correct image. As a result, here we have to check for
-  #   "$UBUNTU_VERSION" == "18.04"*
-  # instead of
-  #   "$UBUNTU_VERSION" == "18.04"
-  if [[ "$UBUNTU_VERSION" == "20.04"* ]]; then
-    cmake3="cmake=3.16*"
-    maybe_libiomp_dev=""
-  elif [[ "$UBUNTU_VERSION" == "22.04"* ]]; then
-    cmake3="cmake=3.22*"
-    maybe_libiomp_dev=""
-  elif [[ "$UBUNTU_VERSION" == "24.04"* ]]; then
-    cmake3="cmake=3.28*"
-    maybe_libiomp_dev=""
-  else
-    cmake3="cmake=3.5*"
-    maybe_libiomp_dev="libiomp-dev"
-  fi
+	install_ubuntu() {
+	  # NVIDIA dockers for RC releases use tag names like `11.0-cudnn9-devel-ubuntu18.04-rc`,
+	  # for this case we will set UBUNTU_VERSION to `18.04-rc` so that the Dockerfile could
+	  # find the correct image. As a result, here we have to check for
+	  #   "$UBUNTU_VERSION" == "18.04"*
+	  # instead of
+	  #   "$UBUNTU_VERSION" == "18.04"
+	  if [[ "$UBUNTU_VERSION" == "20.04"* ]]; then
+	    cmake3="cmake=3.16*"
+	    maybe_libiomp_dev=""
+	  elif [[ "$UBUNTU_VERSION" == "22.04"* ]]; then
+	    cmake3="cmake=3.22*"
+	    maybe_libiomp_dev=""
+	  elif [[ "$UBUNTU_VERSION" == "24.04"* ]]; then
+	    cmake3="cmake=3.28*"
+	    maybe_libiomp_dev=""
+	  else
+	    cmake3="cmake=3.5*"
+	    maybe_libiomp_dev="libiomp-dev"
+	  fi
 
-  if [[ "$CLANG_VERSION" == 15 ]]; then
-    maybe_libomp_dev="libomp-15-dev"
-  elif [[ "$CLANG_VERSION" == 12 ]]; then
-    maybe_libomp_dev="libomp-12-dev"
-  elif [[ "$CLANG_VERSION" == 10 ]]; then
-    maybe_libomp_dev="libomp-10-dev"
-  else
-    maybe_libomp_dev=""
-  fi
+	  if [[ "$CLANG_VERSION" == 15 ]]; then
+	    maybe_libomp_dev="libomp-15-dev"
+	  elif [[ "$CLANG_VERSION" == 12 ]]; then
+	    maybe_libomp_dev="libomp-12-dev"
+	  elif [[ "$CLANG_VERSION" == 10 ]]; then
+	    maybe_libomp_dev="libomp-10-dev"
+	  else
+	    maybe_libomp_dev=""
+	  fi
 
-  # Install common dependencies
-  apt-get update
-  # TODO: Some of these may not be necessary
-  ccache_deps="asciidoc docbook-xml docbook-xsl xsltproc"
-  deploy_deps="libffi-dev libbz2-dev libreadline-dev libncurses5-dev libncursesw5-dev libgdbm-dev libsqlite3-dev uuid-dev tk-dev"
-  numpy_deps="gfortran"
-  apt-get install -y --no-install-recommends \
-    $ccache_deps \
-    $numpy_deps \
-    ${deploy_deps} \
-    ${cmake3} \
-    apt-transport-https \
-    autoconf \
-    automake \
-    build-essential \
-    ca-certificates \
-    curl \
-    git \
-    libatlas-base-dev \
-    libc6-dbg \
-    ${maybe_libiomp_dev} \
-    libyaml-dev \
-    libz-dev \
-    libjemalloc2 \
-    libjpeg-dev \
-    libasound2-dev \
-    libsndfile-dev \
-    ${maybe_libomp_dev} \
-    software-properties-common \
-    wget \
-    sudo \
-    vim \
-    jq \
-    libtool \
-    vim \
-    unzip \
-    gpg-agent \
-    gdb \
-    bc
+	  # Install common dependencies
+	  apt-get update
+	  # TODO: Some of these may not be necessary
+	  ccache_deps="asciidoc docbook-xml docbook-xsl xsltproc"
+	  deploy_deps="libffi-dev libbz2-dev libreadline-dev libncurses5-dev libncursesw5-dev libgdbm-dev libsqlite3-dev uuid-dev tk-dev"
+	  numpy_deps="gfortran"
+	  apt-get install -y --no-install-recommends \
+	    $ccache_deps \
+	    $numpy_deps \
+	    ${deploy_deps} \
+	    ${cmake3} \
+	    apt-transport-https \
+	    autoconf \
+	    automake \
+	    build-essential \
+	    ca-certificates \
+	    curl \
+	    git \
+	    libatlas-base-dev \
+	    libc6-dbg \
+	    ${maybe_libiomp_dev} \
+	    libyaml-dev \
+	    libz-dev \
+	    libjemalloc2 \
+	    libjpeg-dev \
+	    libasound2-dev \
+	    libsndfile-dev \
+	    ${maybe_libomp_dev} \
+	    software-properties-common \
+	    wget \
+	    sudo \
+	    vim \
+	    jq \
+	    libtool \
+	    vim \
+	    unzip \
+	    gpg-agent \
+	    gdb \
+	    bc
 
-  # Should resolve issues related to various apt package repository cert issues
-  # see: https://github.com/pytorch/pytorch/issues/65931
-  apt-get install -y libgnutls30
+	  # Should resolve issues related to various apt package repository cert issues
+	  # see: https://github.com/pytorch/pytorch/issues/65931
+	  apt-get install -y libgnutls30
 
-  # Cleanup package manager
-  apt-get autoclean && apt-get clean
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-}
+	  # Cleanup package manager
+	  apt-get autoclean && apt-get clean
+	  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+	}
 
-install_centos() {
-  # Need EPEL for many packages we depend on.
-  # See http://fedoraproject.org/wiki/EPEL
-  yum --enablerepo=extras install -y epel-release
+	install_centos() {
+	  # Need EPEL for many packages we depend on.
+	  # See http://fedoraproject.org/wiki/EPEL
+	  # extras repo is not there for CentOS 9 and epel-release is already part of repo list
+	  if [[ $OS_VERSION == 9 ]]; then
+	      yum install -y epel-release
+	      ALLOW_ERASE="--allowerasing"
+	  else
+	      yum --enablerepo=extras install -y epel-release
+	      ALLOW_ERASE=""
+	  fi
 
-  ccache_deps="asciidoc docbook-dtds docbook-style-xsl libxslt"
-  numpy_deps="gcc-gfortran"
-  yum install -y \
+	  ccache_deps="asciidoc docbook-dtds docbook-style-xsl libxslt"
+	  numpy_deps="gcc-gfortran"
+	  # Note: protobuf-c-{compiler,devel} on CentOS are too old to be used
+	  # for Caffe2. That said, we still install them to make sure the build
+	  # system opts to build/use protoc and libprotobuf from third-party.
+	  yum install -y $ALLOW_ERASE \
     $ccache_deps \
     $numpy_deps \
     autoconf \
@@ -106,24 +116,34 @@ install_centos() {
     glibc-headers \
     glog-devel \
     libstdc++-devel \
-    libsndfile-devel \
     make \
-    opencv-devel \
     sudo \
     wget \
     vim \
     unzip \
     gdb
 
+  if [[ $OS_VERSION == 9 ]]
+  then
+	  dnf --enablerepo=crb -y install libsndfile-devel
+	  yum install -y procps
+  else
+	  yum install -y \
+            opencv-devel \
+	    libsndfile-devel
+  fi
+
   # Cleanup
   yum clean all
   rm -rf /var/cache/yum
   rm -rf /var/lib/yum/yumdb
   rm -rf /var/lib/yum/history
 }
 
-# Install base packages depending on the base OS
 ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+OS_VERSION=$(grep -oP '(?<=^VERSION_ID=).+' /etc/os-release | tr -d '"')
+
+# Install base packages depending on the base OS
 case "$ID" in
   ubuntu)
     install_ubuntu

.ci/docker/common/install_cache.sh

@@ -36,7 +36,12 @@ sed -e 's|PATH="\(.*\)"|PATH="/opt/cache/bin:\1"|g' -i /etc/environment
 export PATH="/opt/cache/bin:$PATH"
 
 # Setup compiler cache
-install_ubuntu
+if [ -n "$ROCM_VERSION" ]; then
+  curl --retry 3 http://repo.radeon.com/misc/.sccache_amd/sccache -o /opt/cache/bin/sccache
+else
+  install_ubuntu
+fi
+
 chmod a+x /opt/cache/bin/sccache
 
 function write_sccache_stub() {

.ci/docker/common/install_conda.sh

@@ -40,8 +40,13 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
 
   # Prevent conda from updating to 4.14.0, which causes docker build failures
   # See https://hud.pytorch.org/pytorch/pytorch/commit/754d7f05b6841e555cea5a4b2c505dd9e0baec1d
-  # Uncomment the below when resolved to track the latest conda update
-  # as_jenkins conda update -y -n base conda
+  # Uncomment the below when resolved to track the latest conda update,
+  # but this is required for CentOS stream 9 builds
+  ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+  OS_VERSION=$(grep -oP '(?<=^VERSION_ID=).+' /etc/os-release | tr -d '"')
+  if [[ $ID == centos && $OS_VERSION == 9 ]]; then
+    as_jenkins conda update -y -n base conda
+  fi
 
   if [[ $(uname -m) == "aarch64" ]]; then
     export SYSROOT_DEP="sysroot_linux-aarch64=2.17"