Preliminary steps to include ffmpeg in build

Author: mmartial

Makefile

@@ -3,7 +3,7 @@ SHELL := /bin/bash
 .PHONY: all build_all actual_build build_prep
 
 # Release to match data of Dockerfile and follow YYYYMMDD pattern
-CTO_RELEASE=20220530
+CTO_RELEASE=20220815
 
 # The default is not to build OpenCV non-free or build FFmpeg with libnpp, as those would make the images unredistributable 
 # Replace "" by "unredistributable" if you need to use those for a personal build
@@ -44,11 +44,11 @@ DNN_ARCH_CUDA11=6.0,6.1,7.0,7.5,8.0,8.6
 
 # According to https://opencv.org/releases/
 STABLE_OPENCV3=3.4.16
-STABLE_OPENCV4=4.5.5
+STABLE_OPENCV4=4.6.0
 
 # FFmpeg
 # Release list: https://ffmpeg.org/download.html
-# Note: FFmpeg < 5 because https://github.com/pytorch/vision/issues/5928
+# Note: FFmpeg < 5 because https://github.com/opencv/opencv/issues/20147
 # Note: GPU extensions are added directly in the Dockerfile
 CTO_FFMPEG_VERSION=4.4.2
 # https://github.com/FFmpeg/nv-codec-headers/releases
@@ -74,14 +74,14 @@ TF2_NUMPY='numpy'
 # Note: GPU targets (ie ARCH) are directly added in Dockerfile
 CTO_MAGMA="2.6.2"
 
-## PyTorch (with FFmpeg + OpenCV & Magma if available)
+## PyTorch (with FFmpeg + OpenCV & Magma if available) https://pytorch.org/
 # Note: same as FFmpeg and Magma, GPU specific selection (including ARCH) are in the Dockerfile
 # Use release branch https://github.com/pytorch/pytorch
-CTO_TORCH="1.11"
+CTO_TORCH="1.12.1"
 # Use release branch https://github.com/pytorch/vision
-CTO_TORCHVISION="0.12"
+CTO_TORCHVISION="0.13.1"
 # Use release branch https://github.com/pytorch/audio
-CTO_TORCHAUDIO="0.11"
+CTO_TORCHAUDIO="0.12.1"
 
 ##########
 

ubuntu20.04/Dockerfile

@@ -335,8 +335,9 @@ RUN if [ -f /tmp/.GPU_build ]; \
 ARG CTO_TORCH=1.11
 RUN mkdir -p /usr/local/src \
   && cd /usr/local/src \
-  && git clone --depth 1 --branch release/${CTO_TORCH}  https://github.com/pytorch/pytorch.git  \
+  && git clone --depth 1 --branch v${CTO_TORCH} https://github.com/pytorch/pytorch.git  \
   && cd /usr/local/src/pytorch \
+  && git submodule sync && git submodule update --init --recursive \
   && if [ -f /tmp/.GPU_build ]; \
     then \
       time env USE_CUDA=ON USE_CUDNN=ON TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0 7.5 8.0 8.6+PTX" USE_MKLDNN=ON USE_FFMPEG=ON USE_OPENCV=ON python3 ./setup.py bdist_wheel | tee /tmp/torch_config.txt; \
@@ -351,12 +352,23 @@ RUN mkdir -p /usr/local/src \
   && rm -rf /root/.cache/pip /usr/local/src/pytorch
 RUN python3 -c "import torch"
 
+ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES video,compute,utility
+
 ARG CTO_TORCHVISION=0.12
 RUN mkdir -p /usr/local/src \
   && cd /usr/local/src \
-  && git clone --depth 1 --branch release/${CTO_TORCHVISION}  https://github.com/pytorch/vision.git  \
+  && git clone  --depth 1 --branch v${CTO_TORCHVISION} https://github.com/pytorch/vision.git  \
   && cd /usr/local/src/vision \
-  && time sh -c "python3 ./setup.py bdist_wheel | tee /tmp/torchvision_config.txt" \
+  && git submodule sync && git submodule update --init --recursive \
+  && if [ -f /tmp/.GPU_build ]; \
+    then \
+      time env FORCE_CUDA=ON CUDA_HOME=/usr/local/nvidia USE_CUDNN=ON TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0 7.5 8.0 8.6+PTX" USE_MKLDNN=ON USE_FFMPEG=ON USE_OPENCV=ON python3 ./setup.py bdist_wheel | tee /tmp/torchvision_config.txt; \
+    else \
+      time env FORCE_CUDA=OFF USE_CUDNN=OFF USE_MKLDNN=ON USE_FFMPEG=ON USE_OPENCV=ON python3 ./setup.py bdist_wheel | tee /tmp/torchvision_config.txt; \
+    fi \
+#  && time sh -c "python3 ./setup.py bdist_wheel | tee /tmp/torchvision_config.txt" \
   && time pip3 install /usr/local/src/vision/dist/*.whl \
   && cd /tmp \
   && perl -i.bak -pe 'exit if m%^running bdist_wheel%' torchvision_config.txt \
@@ -366,17 +378,24 @@ RUN mkdir -p /usr/local/src \
 RUN python3 -c "import torchvision"
 
 ARG CTO_TORCHAUDIO=0.11
-RUN mkdir -p /usr/local/src/builder \
+RUN mkdir -p /usr/local/src \
   && cd /usr/local/src \
-  && git clone --depth 1 --branch release/${CTO_TORCHAUDIO}  https://github.com/pytorch/audio.git \
+  && git clone --depth 1 --branch v${CTO_TORCHAUDIO} https://github.com/pytorch/audio.git \
   && cd /usr/local/src/audio \
-  && time sh -c "python3 ./setup.py bdist_wheel | tee /tmp/torchaudio_config.txt" \
+  && git submodule sync && git submodule update --init --recursive \
+  && if [ -f /tmp/.GPU_build ]; \
+    then \
+      time env USE_CUDA=ON USE_CUDNN=ON TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0 7.5 8.0 8.6+PTX" USE_MKLDNN=ON USE_FFMPEG=ON USE_OPENCV=ON python3 ./setup.py bdist_wheel | tee /tmp/torchaudio_config.txt; \
+    else \
+      time env USE_CUDA=OFF USE_CUDNN=OFF USE_MKLDNN=ON USE_FFMPEG=ON USE_OPENCV=ON python3 ./setup.py bdist_wheel | tee /tmp/torchaudio_config.txt; \
+    fi \
+#  && time sh -c "python3 ./setup.py bdist_wheel | tee /tmp/torchaudio_config.txt" \
   && time pip3 install /usr/local/src/audio/dist/*.whl \
   && cd /tmp \
   && perl -i.bak -pe 'exit if m%^-- Configuring done%' torchaudio_config.txt \
   && sh -c "cmp --silent torchaudio_config.txt torchaudio_config.txt.bak && exit 1 || rm torchaudio_config.txt.bak" \
   && ldconfig \
-  && rm -rf /root/.cache/pip /usr/local/src/builder
+  && rm -rf /root/.cache/pip /usr/local/src/audio
 RUN python3 -c "import torchaudio"
 
 
@@ -390,11 +409,6 @@ COPY tools/tf_info.sh /tmp/
 RUN chmod +x /tmp/tf_info.sh \
   && touch /.within_container
 
-#ENV NVIDIA_VISIBLE_DEVICES all
-#ENV NVIDIA_DRIVER_CAPABILITIES all
-
-ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH
-
 # Setting up working directory 
 RUN mkdir /dmc
 WORKDIR /dmc