diff --git a/.github/unittest/linux/scripts/environment.yml b/.github/unittest/linux/scripts/environment.yml deleted file mode 100644 index 3283867e9bc..00000000000 --- a/.github/unittest/linux/scripts/environment.yml +++ /dev/null @@ -1,37 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - protobuf - - pip: - - hypothesis - - future - - cloudpickle - - pygame - - moviepy<2.0.0 - - tqdm - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-timeout - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - tensorboard - - imageio==2.26.0 - - wandb - - dm_control - - mujoco<3.3.6 - - mlflow - - av - - coverage - - ray - - transformers - - ninja - - timm diff --git a/.github/unittest/linux/scripts/post_process.sh b/.github/unittest/linux/scripts/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux/scripts/post_process.sh +++ b/.github/unittest/linux/scripts/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux/scripts/requirements.txt b/.github/unittest/linux/scripts/requirements.txt new file mode 100644 index 00000000000..df0a9ac8504 --- /dev/null +++ b/.github/unittest/linux/scripts/requirements.txt @@ -0,0 +1,32 @@ +pip +protobuf +hypothesis +future +cloudpickle +pygame +moviepy<2.0.0 +tqdm +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-timeout +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +tensorboard +imageio==2.26.0 +wandb +dm_control +mujoco<3.3.6 +mlflow +av +coverage +ray +transformers +ninja +timm diff --git a/.github/unittest/linux/scripts/run_all.sh b/.github/unittest/linux/scripts/run_all.sh index 437e2ba7200..f6404974129 100755 --- a/.github/unittest/linux/scripts/run_all.sh +++ b/.github/unittest/linux/scripts/run_all.sh @@ -9,7 +9,7 @@ set -v if [[ $OSTYPE != 'darwin'* ]]; then apt-get update && apt-get upgrade -y - apt-get install -y vim git wget cmake + apt-get install -y vim git wget curl cmake # Enable universe repository # apt-get install -y software-properties-common @@ -27,7 +27,7 @@ if [[ $OSTYPE != 'darwin'* ]]; then apt-get upgrade -y libstdc++6 apt-get dist-upgrade -y else - apt-get install -y g++ gcc + apt-get install -y g++ gcc cmake fi fi @@ -45,37 +45,23 @@ fi # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" -lib_dir="${env_dir}/lib" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac +# 1. Install uv +printf "* Installing uv\n" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" - -# 2. Create test environment at ./env +# 2. Create test environment at ./.venv printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" +printf "* Creating a test environment with uv\n" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 3. Install Conda dependencies +# 3. Install dependencies (except PyTorch) printf "* Installing dependencies (except PyTorch)\n" -echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" -cat "${this_dir}/environment.yml" if [ "${CU_VERSION:-}" == cpu ] ; then export MUJOCO_GL=glfw @@ -85,31 +71,36 @@ fi export SDL_VIDEODRIVER=dummy -# legacy from bash scripts: remove? -conda env config vars set \ - MAX_IDLE_COUNT=1000 \ - MUJOCO_GL=$MUJOCO_GL PYOPENGL_PLATFORM=$MUJOCO_GL DISPLAY=:99 SDL_VIDEODRIVER=dummy LAZY_LEGACY_OP=False RL_LOGGING_LEVEL=DEBUG TOKENIZERS_PARALLELISM=true +# Set environment variables +export MAX_IDLE_COUNT=1000 +export PYOPENGL_PLATFORM=$MUJOCO_GL +export DISPLAY=:99 +export LAZY_LEGACY_OP=False +export RL_LOGGING_LEVEL=DEBUG +export TOKENIZERS_PARALLELISM=true -pip3 install pip --upgrade -pip install virtualenv +# Install build dependencies FIRST (required for C++ extensions) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" -conda env update --file "${this_dir}/environment.yml" --prune +# Install dependencies from requirements.txt +printf "* Installing dependencies from requirements.txt\n" +uv pip install -r "${this_dir}/requirements.txt" -# Reset conda env variables -conda deactivate -conda activate "${env_dir}" +# Install pip for compatibility with packages that expect it +uv pip install pip echo "installing gymnasium" if [[ "$PYTHON_VERSION" == "3.12" ]]; then - pip3 install ale-py - pip3 install sympy - pip3 install "gymnasium[mujoco]>=1.1" mo-gymnasium[mujoco] + uv pip install ale-py + uv pip install sympy + uv pip install "gymnasium[mujoco]>=1.1" mo-gymnasium[mujoco] else - pip3 install "gymnasium[atari,mujoco]>=1.1" mo-gymnasium[mujoco] + uv pip install "gymnasium[atari,mujoco]>=1.1" mo-gymnasium[mujoco] fi # sanity check: remove? -python3 -c """ +python -c """ import dm_control from dm_control import composer from tensorboard import * @@ -140,15 +131,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with %s\n" "${CU_VERSION}" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/$CU_VERSION -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/$CU_VERSION -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu -U + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu -U else - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/$CU_VERSION -U + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/$CU_VERSION -U fi else printf "Failed to install pytorch" @@ -167,23 +158,22 @@ python -c "import functorch" # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi printf "* Installing torchrl\n" -python setup.py develop - +uv pip install -e . --no-build-isolation if [ "${CU_VERSION:-}" != cpu ] ; then printf "* Installing VC1\n" - python3 -c """ + python -c """ from torchrl.envs.transforms.vc1 import VC1Transform VC1Transform.install_vc_models(auto_exit=True) """ - python3 -c """ + python -c """ import vc_models from vc_models.models.vit import model_utils print(model_utils) diff --git a/.github/unittest/linux_distributed/scripts/environment.yml b/.github/unittest/linux_distributed/scripts/environment.yml deleted file mode 100644 index 2eac1112692..00000000000 --- a/.github/unittest/linux_distributed/scripts/environment.yml +++ /dev/null @@ -1,34 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - protobuf - - pip: - - hypothesis - - future - - cloudpickle - - pygame - - moviepy<2.0.0 - - tqdm - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - tensorboard - - imageio==2.26.0 - - wandb - - dm_control - - mujoco<3.3.6 - - mlflow - - av - - coverage - - ray - - virtualenv diff --git a/.github/unittest/linux_distributed/scripts/install.sh b/.github/unittest/linux_distributed/scripts/install.sh index 7caa9a0c6a0..c45eae10e3b 100755 --- a/.github/unittest/linux_distributed/scripts/install.sh +++ b/.github/unittest/linux_distributed/scripts/install.sh @@ -7,8 +7,8 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -28,15 +28,15 @@ git submodule sync && git submodule update --init --recursive if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/$CU_VERSION -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/$CU_VERSION -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu -U + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu -U else - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/$CU_VERSION -U + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/$CU_VERSION -U fi else printf "Failed to install pytorch" @@ -47,14 +47,14 @@ fi python -c "import functorch" ## install snapshot -#pip install git+https://github.com/pytorch/torchsnapshot +#uv pip install git+https://github.com/pytorch/torchsnapshot # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation diff --git a/.github/unittest/linux_distributed/scripts/post_process.sh b/.github/unittest/linux_distributed/scripts/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_distributed/scripts/post_process.sh +++ b/.github/unittest/linux_distributed/scripts/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_distributed/scripts/requirements.txt b/.github/unittest/linux_distributed/scripts/requirements.txt new file mode 100644 index 00000000000..ca9e2a14cc9 --- /dev/null +++ b/.github/unittest/linux_distributed/scripts/requirements.txt @@ -0,0 +1,29 @@ +pip +protobuf +hypothesis +future +cloudpickle +pygame +moviepy<2.0.0 +tqdm +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +tensorboard +imageio==2.26.0 +wandb +dm_control +mujoco<3.3.6 +mlflow +av +coverage +ray +virtualenv diff --git a/.github/unittest/linux_distributed/scripts/run_test.sh b/.github/unittest/linux_distributed/scripts/run_test.sh index 176fefcd73d..593f5d402a4 100755 --- a/.github/unittest/linux_distributed/scripts/run_test.sh +++ b/.github/unittest/linux_distributed/scripts/run_test.sh @@ -2,8 +2,8 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" export PYTORCH_TEST_WITH_SLOW='1' python -m torch.utils.collect_env diff --git a/.github/unittest/linux_distributed/scripts/setup_env.sh b/.github/unittest/linux_distributed/scripts/setup_env.sh index 3681f83ec90..ab0a127e7ef 100755 --- a/.github/unittest/linux_distributed/scripts/setup_env.sh +++ b/.github/unittest/linux_distributed/scripts/setup_env.sh @@ -9,34 +9,28 @@ set -e this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # Avoid error: "fatal: unsafe repository" +apt-get update && apt-get install -y git curl wget gcc g++ cmake git config --global --add safe.directory '*' root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" -lib_dir="${env_dir}/lib" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# 1. Install uv +printf "* Installing uv\n" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" ## 3. Install mujoco #printf "* Installing mujoco and related\n" @@ -63,15 +57,14 @@ else fi export MUJOCO_GL=$PRIVATE_MUJOCO_GL -conda env config vars set MUJOCO_PY_MUJOCO_PATH=$root_dir/.mujoco/mujoco210 \ - MAX_IDLE_COUNT=1000 \ - DISPLAY=:99 \ - MJLIB_PATH=$root_dir/.mujoco/mujoco-2.1.1/lib/libmujoco.so.2.1.1 \ - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$root_dir/.mujoco/mujoco210/bin \ - SDL_VIDEODRIVER=dummy \ - MUJOCO_GL=$PRIVATE_MUJOCO_GL \ - PYOPENGL_PLATFORM=$PRIVATE_MUJOCO_GL \ - TOKENIZERS_PARALLELISM=true +export MUJOCO_PY_MUJOCO_PATH=$root_dir/.mujoco/mujoco210 +export MAX_IDLE_COUNT=1000 +export DISPLAY=:99 +export MJLIB_PATH=$root_dir/.mujoco/mujoco-2.1.1/lib/libmujoco.so.2.1.1 +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$root_dir/.mujoco/mujoco210/bin +export SDL_VIDEODRIVER=dummy +export PYOPENGL_PLATFORM=$PRIVATE_MUJOCO_GL +export TOKENIZERS_PARALLELISM=true # Software rendering requires GLX and OSMesa. if [ $PRIVATE_MUJOCO_GL == 'egl' ] || [ $PRIVATE_MUJOCO_GL == 'osmesa' ] ; then @@ -85,19 +78,17 @@ if [ $PRIVATE_MUJOCO_GL == 'egl' ] || [ $PRIVATE_MUJOCO_GL == 'osmesa' ] ; then yum -y install freeglut fi -pip install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) -conda deactivate -conda activate "${env_dir}" if [[ $OSTYPE != 'darwin'* ]]; then # install ale-py: manylinux names are broken for CentOS so we need to manually download and # rename them PY_VERSION=$(python --version) echo "installing gymnasium" - pip install "gymnasium[atari]>=1.1" + uv pip install "gymnasium[atari]>=1.1" else - pip install "gymnasium[atari]>=1.1" + uv pip install "gymnasium[atari]>=1.1" fi diff --git a/.github/unittest/linux_libs/scripts_ataridqn/environment.yml b/.github/unittest/linux_libs/scripts_ataridqn/environment.yml deleted file mode 100644 index ea3ec1d0328..00000000000 --- a/.github/unittest/linux_libs/scripts_ataridqn/environment.yml +++ /dev/null @@ -1,27 +0,0 @@ -channels: - - pytorch - - defaults - - conda-forge -dependencies: - - pip - - gsutil - - pip: - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - tqdm - - h5py - - datasets - - pillow diff --git a/.github/unittest/linux_libs/scripts_ataridqn/install.sh b/.github/unittest/linux_libs/scripts_ataridqn/install.sh index 0c5e0ab636f..5ed6b58e905 100755 --- a/.github/unittest/linux_libs/scripts_ataridqn/install.sh +++ b/.github/unittest/linux_libs/scripts_ataridqn/install.sh @@ -7,8 +7,14 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +# Add uv to PATH (it was installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -29,15 +35,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu else - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -46,16 +52,16 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import functorch;import tensordict" printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation # smoke test python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_ataridqn/post_process.sh b/.github/unittest/linux_libs/scripts_ataridqn/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_ataridqn/post_process.sh +++ b/.github/unittest/linux_libs/scripts_ataridqn/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_ataridqn/requirements.txt b/.github/unittest/linux_libs/scripts_ataridqn/requirements.txt new file mode 100644 index 00000000000..46d70854669 --- /dev/null +++ b/.github/unittest/linux_libs/scripts_ataridqn/requirements.txt @@ -0,0 +1,21 @@ +pip +gsutil +hypothesis +future +cloudpickle +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +tqdm +h5py +datasets +pillow diff --git a/.github/unittest/linux_libs/scripts_ataridqn/run_test.sh b/.github/unittest/linux_libs/scripts_ataridqn/run_test.sh index 8b7a1858f14..d8cee3c17ac 100755 --- a/.github/unittest/linux_libs/scripts_ataridqn/run_test.sh +++ b/.github/unittest/linux_libs/scripts_ataridqn/run_test.sh @@ -2,8 +2,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" apt-get update && apt-get remove swig -y && apt-get install -y git gcc patchelf libosmesa6-dev libgl1-mesa-glx libglfw3 swig3.0 cmake ln -s /usr/bin/swig3.0 /usr/bin/swig @@ -18,7 +19,7 @@ root_dir="$(git rev-parse --show-toplevel)" env_dir="${root_dir}/env" lib_dir="${env_dir}/lib" -conda deactivate && conda activate ./env +# conda deactivate (not needed with uv) && source ./.venv/bin/activate python .github/unittest/helpers/coverage_run_parallel.py -m pytest test/test_libs.py --instafail -v --durations 200 --capture no -k TestAtariDQN --error-for-skips --runslow coverage combine diff --git a/.github/unittest/linux_libs/scripts_ataridqn/setup_env.sh b/.github/unittest/linux_libs/scripts_ataridqn/setup_env.sh index 8739dc861fd..afcdfce59d2 100755 --- a/.github/unittest/linux_libs/scripts_ataridqn/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_ataridqn/setup_env.sh @@ -12,7 +12,7 @@ set -v apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -34,37 +34,32 @@ apt-get upgrade -y libstdc++6 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" # 3. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" -pip3 install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) diff --git a/.github/unittest/linux_libs/scripts_brax/environment.yml b/.github/unittest/linux_libs/scripts_brax/environment.yml deleted file mode 100644 index 63e26428d55..00000000000 --- a/.github/unittest/linux_libs/scripts_brax/environment.yml +++ /dev/null @@ -1,24 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - jax[cuda12]>=0.7.0 - - brax - - psutil diff --git a/.github/unittest/linux_libs/scripts_brax/install.sh b/.github/unittest/linux_libs/scripts_brax/install.sh old mode 100755 new mode 100644 index 0f90a5e4210..3a5df02d5dc --- a/.github/unittest/linux_libs/scripts_brax/install.sh +++ b/.github/unittest/linux_libs/scripts_brax/install.sh @@ -7,8 +7,16 @@ unset PYTORCH_VERSION set -euxo pipefail -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +# Make uv available (installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" + +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -28,15 +36,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch --index-url https://download.pytorch.org/whl/cpu -U + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu -U else - pip3 install torch --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -44,13 +52,14 @@ else fi # install tensordict -pip install git+https://github.com/pytorch/tensordict.git --progress-bar off +uv pip install git+https://github.com/pytorch/tensordict.git off # smoke test python -c "import functorch;import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation # smoke test python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_brax/post_process.sh b/.github/unittest/linux_libs/scripts_brax/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_brax/post_process.sh +++ b/.github/unittest/linux_libs/scripts_brax/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_brax/requirements.txt b/.github/unittest/linux_libs/scripts_brax/requirements.txt new file mode 100644 index 00000000000..fd09e7d5eb7 --- /dev/null +++ b/.github/unittest/linux_libs/scripts_brax/requirements.txt @@ -0,0 +1,19 @@ +pip +hypothesis +future +cloudpickle +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +jax[cuda12]>=0.7.0 +brax +psutil diff --git a/.github/unittest/linux_libs/scripts_brax/run_test.sh b/.github/unittest/linux_libs/scripts_brax/run_test.sh index 2fb6f2465fd..4eed47d49da 100755 --- a/.github/unittest/linux_libs/scripts_brax/run_test.sh +++ b/.github/unittest/linux_libs/scripts_brax/run_test.sh @@ -2,8 +2,9 @@ set -euxo pipefail -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" export PYTORCH_TEST_WITH_SLOW='1' @@ -62,7 +63,7 @@ except Exception as e: print('Falling back to JAX CPU') " -python3 -c 'import torch;t = torch.ones([2,2], device="cuda:0");print(t);print("tensor device:" + str(t.device))' +python -c 'import torch;t = torch.ones([2,2], device="cuda:0");print(t);print("tensor device:" + str(t.device))' python .github/unittest/helpers/coverage_run_parallel.py -m pytest test/test_libs.py --instafail -v --durations 200 --capture no -k TestBrax --error-for-skips coverage combine diff --git a/.github/unittest/linux_libs/scripts_brax/setup_env.sh b/.github/unittest/linux_libs/scripts_brax/setup_env.sh index edbb11de15d..f43e1f15ed6 100755 --- a/.github/unittest/linux_libs/scripts_brax/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_brax/setup_env.sh @@ -10,7 +10,7 @@ set -euxo pipefail apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -32,50 +32,28 @@ apt-get upgrade -y libstdc++6 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac +# 1. Install uv +printf "* Installing uv\n" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" - -# 2. Create test environment at ./env +# 2. Create test environment at ./.venv printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" - -## 3. Install mujoco -#printf "* Installing mujoco and related\n" -#mkdir $root_dir/.mujoco -#cd $root_dir/.mujoco/ -#wget https://github.com/deepmind/mujoco/releases/download/2.1.1/mujoco-2.1.1-linux-x86_64.tar.gz -#tar -xf mujoco-2.1.1-linux-x86_64.tar.gz -#wget https://mujoco.org/download/mujoco210-linux-x86_64.tar.gz -#tar -xf mujoco210-linux-x86_64.tar.gz -#cd $this_dir - -# 4. Install Conda dependencies -printf "* Installing dependencies (except PyTorch)\n" -echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" -cat "${this_dir}/environment.yml" +printf "* Creating a test environment with uv\n" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -pip install pip --upgrade +# 3. Install build dependencies FIRST (required for C++ extensions) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" -conda env update --file "${this_dir}/environment.yml" --prune +# 4. Install test dependencies and libraries (except PyTorch) +printf "* Installing dependencies from requirements.txt\n" +uv pip install -r "${this_dir}/requirements.txt" #yum makecache # sudo yum -y install glfw diff --git a/.github/unittest/linux_libs/scripts_chess/environment.yml b/.github/unittest/linux_libs/scripts_chess/environment.yml deleted file mode 100644 index bf1e689c516..00000000000 --- a/.github/unittest/linux_libs/scripts_chess/environment.yml +++ /dev/null @@ -1,25 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - chess - - transformers - - cairosvg - - pycairo diff --git a/.github/unittest/linux_libs/scripts_chess/install.sh b/.github/unittest/linux_libs/scripts_chess/install.sh old mode 100755 new mode 100644 index 2339fb14bde..f694c03573c --- a/.github/unittest/linux_libs/scripts_chess/install.sh +++ b/.github/unittest/linux_libs/scripts_chess/install.sh @@ -24,8 +24,15 @@ apt-get update && apt-get install -y \ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +# Add uv to PATH (it was installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -45,15 +52,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu else - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -62,16 +69,17 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import functorch;import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation # smoke test python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_chess/post_process.sh b/.github/unittest/linux_libs/scripts_chess/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_chess/post_process.sh +++ b/.github/unittest/linux_libs/scripts_chess/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_chess/requirements.txt b/.github/unittest/linux_libs/scripts_chess/requirements.txt new file mode 100644 index 00000000000..cefa4686810 --- /dev/null +++ b/.github/unittest/linux_libs/scripts_chess/requirements.txt @@ -0,0 +1,20 @@ +pip +hypothesis +future +cloudpickle +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +chess +transformers +cairosvg +pycairo diff --git a/.github/unittest/linux_libs/scripts_chess/run_test.sh b/.github/unittest/linux_libs/scripts_chess/run_test.sh index dbf769e6ee9..c4217ea626b 100755 --- a/.github/unittest/linux_libs/scripts_chess/run_test.sh +++ b/.github/unittest/linux_libs/scripts_chess/run_test.sh @@ -2,8 +2,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" apt-get update && apt-get install -y git wget cmake @@ -17,7 +18,7 @@ root_dir="$(git rev-parse --show-toplevel)" env_dir="${root_dir}/env" lib_dir="${env_dir}/lib" -conda deactivate && conda activate ./env +# conda deactivate (not needed with uv) && source ./.venv/bin/activate # this workflow only tests the libs python -c "import chess" diff --git a/.github/unittest/linux_libs/scripts_chess/setup_env.sh b/.github/unittest/linux_libs/scripts_chess/setup_env.sh index e62891418bd..21fd966ffc0 100755 --- a/.github/unittest/linux_libs/scripts_chess/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_chess/setup_env.sh @@ -12,7 +12,7 @@ this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -31,40 +31,34 @@ apt-get install -y wget \ libcairo2-dev root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" # 3. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" -pip install pip --upgrade +uv pip install pip --upgrade -conda install anaconda::cmake -y -conda install conda-forge::cairo -y +uv pip install cairo -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) diff --git a/.github/unittest/linux_libs/scripts_d4rl/environment.yml b/.github/unittest/linux_libs/scripts_d4rl/environment.yml deleted file mode 100644 index fa74af321dc..00000000000 --- a/.github/unittest/linux_libs/scripts_d4rl/environment.yml +++ /dev/null @@ -1,22 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - cython<3 diff --git a/.github/unittest/linux_libs/scripts_d4rl/install.sh b/.github/unittest/linux_libs/scripts_d4rl/install.sh old mode 100755 new mode 100644 index 0094f045932..c43fab106c6 --- a/.github/unittest/linux_libs/scripts_d4rl/install.sh +++ b/.github/unittest/linux_libs/scripts_d4rl/install.sh @@ -7,8 +7,15 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +# Add uv to PATH (it was installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -29,15 +36,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch --index-url https://download.pytorch.org/whl/cpu + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu else - pip3 install torch --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -46,16 +53,17 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import functorch;import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation # smoke test python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_d4rl/post_process.sh b/.github/unittest/linux_libs/scripts_d4rl/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_d4rl/post_process.sh +++ b/.github/unittest/linux_libs/scripts_d4rl/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_d4rl/requirements.txt b/.github/unittest/linux_libs/scripts_d4rl/requirements.txt new file mode 100644 index 00000000000..f992c77a254 --- /dev/null +++ b/.github/unittest/linux_libs/scripts_d4rl/requirements.txt @@ -0,0 +1,17 @@ +pip +hypothesis +future +cloudpickle +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +cython<3 diff --git a/.github/unittest/linux_libs/scripts_d4rl/run_test.sh b/.github/unittest/linux_libs/scripts_d4rl/run_test.sh old mode 100755 new mode 100644 index dfa14393249..93144c251c3 --- a/.github/unittest/linux_libs/scripts_d4rl/run_test.sh +++ b/.github/unittest/linux_libs/scripts_d4rl/run_test.sh @@ -2,8 +2,14 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + apt-get update && apt-get remove swig -y && apt-get install -y git gcc patchelf libosmesa6-dev libgl1-mesa-glx libglfw3 swig3.0 cmake ln -s /usr/bin/swig3.0 /usr/bin/swig @@ -11,16 +17,16 @@ ln -s /usr/bin/swig3.0 /usr/bin/swig # we install d4rl here bc env variables have been updated git clone https://github.com/Farama-Foundation/d4rl.git cd d4rl -#pip3 install -U 'mujoco-py<2.1,>=2.0' -pip3 install -U "gym[classic_control,atari,accept-rom-license]"==0.23 -pip3 install -U six -pip install -e . +#uv pip install -U 'mujoco-py<2.1,>=2.0' +uv pip install -U "gym[classic_control,atari,accept-rom-license]"==0.23 +uv pip install -U six +uv pip install -e . --no-build-isolation cd .. #flow is a dependency disaster of biblical scale #git clone https://github.com/flow-project/flow.git #cd flow -#python setup.py develop +#uv pip install -e . --no-build-isolation #cd .. export PYTORCH_TEST_WITH_SLOW='1' @@ -33,7 +39,7 @@ root_dir="$(git rev-parse --show-toplevel)" env_dir="${root_dir}/env" lib_dir="${env_dir}/lib" -conda deactivate && conda activate ./env +# conda deactivate (not needed with uv) && source ./.venv/bin/activate # this workflow only tests the libs printf "* Smoke test\n" diff --git a/.github/unittest/linux_libs/scripts_d4rl/setup_env.sh b/.github/unittest/linux_libs/scripts_d4rl/setup_env.sh index bd421dcd0f7..92903055876 100755 --- a/.github/unittest/linux_libs/scripts_d4rl/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_d4rl/setup_env.sh @@ -12,7 +12,7 @@ this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -37,38 +37,33 @@ ln -s libglut.so.3.12 libglut.so.3 cd $this_dir root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" -python3 -m pip install pip --upgrade +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" + + +uv pip install pip --upgrade #pip3 uninstall cython -y #pip uninstall cython -y #conda uninstall cython -y -python3 -m pip install "cython<3" --upgrade +uv pip install "cython<3" --upgrade #conda install -c anaconda cython="<3.0.0" -y @@ -83,19 +78,22 @@ wget https://pytorch.s3.amazonaws.com/torchrl/github-artifacts/mujoco200_linux.z unzip mujoco200_linux.zip wget https://pytorch.s3.amazonaws.com/torchrl/github-artifacts/mjkey.txt cp mjkey.txt ./mujoco200_linux/bin/ +# 4. Install build dependencies FIRST (required for C++ extensions AND mujoco-py) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + # install mujoco-py locally git clone https://github.com/vmoens/mujoco-py.git cd mujoco-py git checkout v2.0.2.1 -pip install -e . +# Install poetry for mujoco-py build (it uses poetry as build backend) +uv pip install poetry +uv pip install -e . --no-build-isolation cd $this_dir -# 4. Install Conda dependencies -printf "* Installing dependencies (except PyTorch)\n" -echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" -cat "${this_dir}/environment.yml" - -pip3 install pip --upgrade +# 5. Install test dependencies and libraries (except PyTorch) +printf "* Installing dependencies from requirements.txt\n" +uv pip install -r "${this_dir}/requirements.txt" setuptools # 5. env variables if [[ $OSTYPE == 'darwin'* ]]; then @@ -107,9 +105,7 @@ else fi export MUJOCO_GL=$PRIVATE_MUJOCO_GL -conda env config vars set \ - MAX_IDLE_COUNT=1000 \ - MUJOCO_PY_MUJOCO_PATH=$root_dir/.mujoco/mujoco200_linux \ +export MAX_IDLE_COUNT=1000 MUJOCO_PY_MUJOCO_PATH=$root_dir/.mujoco/mujoco200_linux \ DISPLAY=:99 \ MJLIB_PATH=$root_dir/.mujoco/mujoco200_linux/bin/libmujoco200.so \ LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$root_dir/.mujoco/mujoco200_linux/bin \ @@ -121,4 +117,4 @@ conda env config vars set \ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libstdc++.so.6 -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) diff --git a/.github/unittest/linux_libs/scripts_envpool/environment.yml b/.github/unittest/linux_libs/scripts_envpool/environment.yml deleted file mode 100644 index 33bccfbab6e..00000000000 --- a/.github/unittest/linux_libs/scripts_envpool/environment.yml +++ /dev/null @@ -1,25 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - protobuf - - pip: - - hypothesis - - future - - cloudpickle - - pygame - - moviepy<2.0.0 - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - dm_control - - mujoco<3.3.6 - - coverage diff --git a/.github/unittest/linux_libs/scripts_envpool/install.sh b/.github/unittest/linux_libs/scripts_envpool/install.sh index 52542e4c299..624eee7d24d 100755 --- a/.github/unittest/linux_libs/scripts_envpool/install.sh +++ b/.github/unittest/linux_libs/scripts_envpool/install.sh @@ -7,8 +7,14 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +# Add uv to PATH (it was installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -28,16 +34,16 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi # smoke test python -c "import functorch" # install tensordict -pip install git+https://github.com/pytorch/tensordict +uv pip install git+https://github.com/pytorch/tensordict printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation diff --git a/.github/unittest/linux_libs/scripts_envpool/post_process.sh b/.github/unittest/linux_libs/scripts_envpool/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_envpool/post_process.sh +++ b/.github/unittest/linux_libs/scripts_envpool/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_envpool/requirements.txt b/.github/unittest/linux_libs/scripts_envpool/requirements.txt new file mode 100644 index 00000000000..925b158550c --- /dev/null +++ b/.github/unittest/linux_libs/scripts_envpool/requirements.txt @@ -0,0 +1,20 @@ +pip +protobuf +hypothesis +future +cloudpickle +pygame +moviepy<2.0.0 +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy +dm_control +mujoco<3.3.6 +coverage diff --git a/.github/unittest/linux_libs/scripts_envpool/run_test.sh b/.github/unittest/linux_libs/scripts_envpool/run_test.sh index bb4ee655673..3c5a67bfc71 100755 --- a/.github/unittest/linux_libs/scripts_envpool/run_test.sh +++ b/.github/unittest/linux_libs/scripts_envpool/run_test.sh @@ -8,8 +8,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" export PYTORCH_TEST_WITH_SLOW='1' export LAZY_LEGACY_OP=False diff --git a/.github/unittest/linux_libs/scripts_envpool/setup_env.sh b/.github/unittest/linux_libs/scripts_envpool/setup_env.sh index 49ac816c3a8..d486c020397 100755 --- a/.github/unittest/linux_libs/scripts_envpool/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_envpool/setup_env.sh @@ -9,7 +9,7 @@ set -e apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -32,49 +32,39 @@ apt-get upgrade -y libstdc++6 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" -lib_dir="${env_dir}/lib" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" # 3. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" -pip install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) apt update -conda env config vars set \ - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libstdc++.so.6 \ - MUJOCO_GL=egl +export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libstdc++.so.6 MUJOCO_GL=egl -conda deactivate -conda activate "${env_dir}" if [[ $OSTYPE != 'darwin'* ]]; then # install ale-py: manylinux names are broken for CentOS so we need to manually download and @@ -83,27 +73,27 @@ if [[ $OSTYPE != 'darwin'* ]]; then echo "installing ale-py for ${PY_PY_VERSION}" if [[ $PY_VERSION == *"3.9"* ]]; then wget https://files.pythonhosted.org/packages/a0/98/4316c1cedd9934f9a91b6e27a9be126043b4445594b40cfa391c8de2e5e8/ale_py-0.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pip install ale_py-0.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + uv pip install ale_py-0.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl rm ale_py-0.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl elif [[ $PY_VERSION == *"3.10"* ]]; then wget https://files.pythonhosted.org/packages/60/1b/3adde7f44f79fcc50d0a00a0643255e48024c4c3977359747d149dc43500/ale_py-0.8.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl mv ale_py-0.8.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl ale_py-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pip install ale_py-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + uv pip install ale_py-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl rm ale_py-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl elif [[ $PY_VERSION == *"3.11"* ]]; then wget https://files.pythonhosted.org/packages/60/1b/3adde7f44f79fcc50d0a00a0643255e48024c4c3977359747d149dc43500/ale_py-0.8.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl mv ale_py-0.8.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl ale_py-0.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pip install ale_py-0.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + uv pip install ale_py-0.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl rm ale_py-0.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl fi echo "installing gym" # envpool does not currently work with gymnasium - pip install "gym[atari,accept-rom-license]<1.0" + uv pip install "gym[atari,accept-rom-license]<1.0" else - pip install "gym[atari,accept-rom-license]<1.0" + uv pip install "gym[atari,accept-rom-license]<1.0" fi -pip install envpool treevalue +uv pip install envpool treevalue # wget https://files.pythonhosted.org/packages/a1/ee/b35ab21e71d34cdcedf05c0d32abca3a3e87d669bdc772660165ee2f55b8/envpool-0.8.2-cp39-cp39-manylinux_2_24_x86_64.whl -# pip install envpool-0.8.2-cp39-cp39-manylinux_2_24_x86_64.whl +# uv pip install envpool-0.8.2-cp39-cp39-manylinux_2_24_x86_64.whl # rm envpool-0.8.2-cp39-cp39-manylinux_2_24_x86_64.whl diff --git a/.github/unittest/linux_libs/scripts_gen-dgrl/environment.yml b/.github/unittest/linux_libs/scripts_gen-dgrl/environment.yml deleted file mode 100644 index 258ee278448..00000000000 --- a/.github/unittest/linux_libs/scripts_gen-dgrl/environment.yml +++ /dev/null @@ -1,23 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - huggingface_hub - - tqdm diff --git a/.github/unittest/linux_libs/scripts_gen-dgrl/install.sh b/.github/unittest/linux_libs/scripts_gen-dgrl/install.sh old mode 100755 new mode 100644 index 0c5e0ab636f..c43fab106c6 --- a/.github/unittest/linux_libs/scripts_gen-dgrl/install.sh +++ b/.github/unittest/linux_libs/scripts_gen-dgrl/install.sh @@ -7,8 +7,15 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +# Add uv to PATH (it was installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -29,15 +36,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu else - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -46,16 +53,17 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import functorch;import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation # smoke test python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_gen-dgrl/post_process.sh b/.github/unittest/linux_libs/scripts_gen-dgrl/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_gen-dgrl/post_process.sh +++ b/.github/unittest/linux_libs/scripts_gen-dgrl/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_gen-dgrl/requirements.txt b/.github/unittest/linux_libs/scripts_gen-dgrl/requirements.txt new file mode 100644 index 00000000000..90575c9503b --- /dev/null +++ b/.github/unittest/linux_libs/scripts_gen-dgrl/requirements.txt @@ -0,0 +1,18 @@ +pip +hypothesis +future +cloudpickle +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +huggingface_hub +tqdm diff --git a/.github/unittest/linux_libs/scripts_gen-dgrl/run_test.sh b/.github/unittest/linux_libs/scripts_gen-dgrl/run_test.sh index 8d53f4e6a58..6e733e7a71b 100755 --- a/.github/unittest/linux_libs/scripts_gen-dgrl/run_test.sh +++ b/.github/unittest/linux_libs/scripts_gen-dgrl/run_test.sh @@ -2,8 +2,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" apt-get update && apt-get remove swig -y && apt-get install -y git gcc patchelf libosmesa6-dev libgl1-mesa-glx libglfw3 swig3.0 cmake ln -s /usr/bin/swig3.0 /usr/bin/swig @@ -18,7 +19,7 @@ root_dir="$(git rev-parse --show-toplevel)" env_dir="${root_dir}/env" lib_dir="${env_dir}/lib" -conda deactivate && conda activate ./env +# conda deactivate (not needed with uv) && source ./.venv/bin/activate python .github/unittest/helpers/coverage_run_parallel.py -m pytest test/test_libs.py --instafail -v --durations 200 --capture no -k TestGenDGRL --error-for-skips --runslow coverage combine diff --git a/.github/unittest/linux_libs/scripts_gen-dgrl/setup_env.sh b/.github/unittest/linux_libs/scripts_gen-dgrl/setup_env.sh index f3df1452583..a3e3e7b3297 100755 --- a/.github/unittest/linux_libs/scripts_gen-dgrl/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_gen-dgrl/setup_env.sh @@ -11,7 +11,7 @@ set -v apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -34,37 +34,32 @@ apt-get upgrade -y libstdc++6 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" # 3. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" -pip3 install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) diff --git a/.github/unittest/linux_libs/scripts_gym/batch_scripts.sh b/.github/unittest/linux_libs/scripts_gym/batch_scripts.sh index 6838afe2be7..8a24928ae21 100755 --- a/.github/unittest/linux_libs/scripts_gym/batch_scripts.sh +++ b/.github/unittest/linux_libs/scripts_gym/batch_scripts.sh @@ -8,8 +8,8 @@ DIR="$(cd "$(dirname "$0")" && pwd)" set -e set -v -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" # Install PyTorch and TorchRL. $DIR/install.sh @@ -18,147 +18,66 @@ $DIR/install.sh apt-get update && apt-get install -y git wget libglew-dev libx11-dev x11proto-dev g++ # solves "'extras_require' must be a dictionary" -pip install setuptools==65.3.0 - -#mkdir -p third_party -#cd third_party -#git clone https://github.com/vmoens/gym -#cd .. - -# This version is installed initially (see environment.yml) -for GYM_VERSION in '0.13' -do - # Create a copy of the conda env and work with this - conda deactivate - conda create --prefix ./cloned_env --clone ./env -y - conda activate ./cloned_env +uv pip install setuptools==65.3.0 +# Helper function to test with a specific gym version +test_gym_version() { + local GYM_VERSION=$1 + local EXTRA_INSTALL=$2 + echo "Testing gym version: ${GYM_VERSION}" - pip3 install 'gym[atari]'==$GYM_VERSION + + # Install the specific gym version in the current environment + eval "$EXTRA_INSTALL" + $DIR/run_test.sh + + # Uninstall gym to avoid conflicts with next version + uv pip uninstall gym gymnasium -y 2>/dev/null || true +} - # delete the conda copy - conda deactivate - conda env remove --prefix ./cloned_env -y +# This version is installed initially +for GYM_VERSION in '0.13' +do + test_gym_version "$GYM_VERSION" "uv pip install 'gym[atari]==$GYM_VERSION'" done # gym[atari]==0.19 is broken, so we install only gym without dependencies. for GYM_VERSION in '0.19' do - # Create a copy of the conda env and work with this - conda deactivate - conda create --prefix ./cloned_env --clone ./env -y - conda activate ./cloned_env - - echo "Testing gym version: ${GYM_VERSION}" - # handling https://github.com/openai/gym/issues/3202 - pip3 install wheel==0.38.4 - pip3 install "pip<24.1" - pip3 install gym==$GYM_VERSION - $DIR/run_test.sh - - # delete the conda copy - conda deactivate - conda env remove --prefix ./cloned_env -y + test_gym_version "$GYM_VERSION" "uv pip install wheel==0.38.4 && uv pip install 'pip<24.1' && uv pip install gym==$GYM_VERSION" done # gym[atari]==0.20 installs ale-py==0.8, but this version is not compatible with gym<0.26, so we downgrade it. for GYM_VERSION in '0.20' do - # Create a copy of the conda env and work with this - conda deactivate - conda create --prefix ./cloned_env --clone ./env -y - conda activate ./cloned_env - - echo "Testing gym version: ${GYM_VERSION}" - pip3 install wheel==0.38.4 - pip3 install "pip<24.1" - pip3 install 'gym[atari]'==$GYM_VERSION - pip3 install ale-py==0.7 - $DIR/run_test.sh - - # delete the conda copy - conda deactivate - conda env remove --prefix ./cloned_env -y + test_gym_version "$GYM_VERSION" "uv pip install wheel==0.38.4 && uv pip install 'pip<24.1' && uv pip install 'gym[atari]==$GYM_VERSION' && uv pip install ale-py==0.7" done for GYM_VERSION in '0.25' do - # Create a copy of the conda env and work with this - conda deactivate - conda create --prefix ./cloned_env --clone ./env -y - conda activate ./cloned_env - - echo "Testing gym version: ${GYM_VERSION}" - pip3 install 'gym[atari]'==$GYM_VERSION - pip3 install pip -U - $DIR/run_test.sh - - # delete the conda copy - conda deactivate - conda env remove --prefix ./cloned_env -y + test_gym_version "$GYM_VERSION" "uv pip install 'gym[atari]==$GYM_VERSION'" done # For this version "gym[accept-rom-license]" is required. for GYM_VERSION in '0.26' do - # Create a copy of the conda env and work with this - conda deactivate - conda create --prefix ./cloned_env --clone ./env -y - conda activate ./cloned_env - - echo "Testing gym version: ${GYM_VERSION}" - pip3 install 'gym[atari,accept-rom-license]'==$GYM_VERSION - pip3 install gym-super-mario-bros - $DIR/run_test.sh - - # delete the conda copy - conda deactivate - conda env remove --prefix ./cloned_env -y + test_gym_version "$GYM_VERSION" "uv pip install 'gym[atari,accept-rom-license]==$GYM_VERSION' && uv pip install gym-super-mario-bros" done # For this version "gym[accept-rom-license]" is required. for GYM_VERSION in '0.27' '0.28' do - # Create a copy of the conda env and work with this - conda deactivate - conda create --prefix ./cloned_env --clone ./env -y - conda activate ./cloned_env - - echo "Testing gym version: ${GYM_VERSION}" - pip3 install 'gymnasium[atari,ale-py]'==$GYM_VERSION - - $DIR/run_test.sh - - # delete the conda copy - conda deactivate - conda env remove --prefix ./cloned_env -y + test_gym_version "$GYM_VERSION" "uv pip install 'gymnasium[atari,ale-py]==$GYM_VERSION'" done # Prev gymnasium -conda deactivate -conda create --prefix ./cloned_env --clone ./env -y -conda activate ./cloned_env - -pip3 install 'gymnasium[ale-py,atari]>=1.1.0' mo-gymnasium gymnasium-robotics -U - +echo "Testing gymnasium >= 1.1.0" +uv pip install 'gymnasium[ale-py,atari]>=1.1.0' mo-gymnasium gymnasium-robotics -U $DIR/run_test.sh - -# delete the conda copy -conda deactivate -conda env remove --prefix ./cloned_env -y - -# Skip 1.0.0 +uv pip uninstall gymnasium -y 2>/dev/null || true # Latest gymnasium -conda deactivate -conda create --prefix ./cloned_env --clone ./env -y -conda activate ./cloned_env - -pip3 install 'gymnasium[ale-py,atari]>=1.1.0' mo-gymnasium gymnasium-robotics -U - +echo "Testing latest gymnasium" +uv pip install 'gymnasium[ale-py,atari]>=1.1.0' mo-gymnasium gymnasium-robotics -U $DIR/run_test.sh - -# delete the conda copy -conda deactivate -conda env remove --prefix ./cloned_env -y diff --git a/.github/unittest/linux_libs/scripts_gym/environment.yml b/.github/unittest/linux_libs/scripts_gym/environment.yml deleted file mode 100644 index 09cef01dec2..00000000000 --- a/.github/unittest/linux_libs/scripts_gym/environment.yml +++ /dev/null @@ -1,29 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - protobuf - - pip: - # Initial version is required to install Atari ROMS in setup_env.sh - - gym[atari]==0.13 - - hypothesis - - future - - cloudpickle - - pygame - - moviepy<2.0.0 - - tqdm - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - patchelf - - pyopengl==3.1.0 diff --git a/.github/unittest/linux_libs/scripts_gym/install.sh b/.github/unittest/linux_libs/scripts_gym/install.sh old mode 100755 new mode 100644 index aff5a714beb..4595dbcaf4c --- a/.github/unittest/linux_libs/scripts_gym/install.sh +++ b/.github/unittest/linux_libs/scripts_gym/install.sh @@ -8,8 +8,16 @@ apt-get update && apt-get install -y git wget gcc g++ set -e set -v -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +# Make uv available (installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" + +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + #apt-get update -y && apt-get install git wget gcc g++ -y @@ -37,28 +45,29 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with %s\n" "${CU_VERSION}" if [ "${CU_VERSION:-}" == cpu ] ; then - conda install pytorch==2.0 torchvision==0.15 cpuonly -c pytorch -y + uv pip install torch==2.0 torchvision==0.15 --index-url https://download.pytorch.org/whl/cpu else - conda install pytorch==2.0.1 torchvision==0.15.2 torchaudio==2.0.2 pytorch-cuda=11.8 numpy==1.26 numpy-base==1.26 -c pytorch -c nvidia -y + uv pip install torch==2.0.1 torchvision==0.15.2 torchaudio==2.0.2 numpy==1.26 --index-url https://download.pytorch.org/whl/cu118 fi # Solving circular import: https://stackoverflow.com/questions/75501048/how-to-fix-attributeerror-partially-initialized-module-charset-normalizer-has -pip install -U charset-normalizer +uv pip install -U charset-normalizer # install tensordict if [[ "$RELEASE" == 0 ]]; then - conda install "anaconda::cmake>=3.22" -y - pip3 install "pybind11[global]" - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install "cmake>=3.22" + uv pip install "pybind11[global]" + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation python -c "import torchrl" ## Reinstalling pytorch with specific version diff --git a/.github/unittest/linux_libs/scripts_gym/post_process.sh b/.github/unittest/linux_libs/scripts_gym/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_gym/post_process.sh +++ b/.github/unittest/linux_libs/scripts_gym/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_gym/requirements.txt b/.github/unittest/linux_libs/scripts_gym/requirements.txt new file mode 100644 index 00000000000..8c8aed412ba --- /dev/null +++ b/.github/unittest/linux_libs/scripts_gym/requirements.txt @@ -0,0 +1,23 @@ +pip +protobuf +gym[atari]==0.13 +hypothesis +future +cloudpickle +pygame +moviepy<2.0.0 +tqdm +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +patchelf +pyopengl==3.1.0 diff --git a/.github/unittest/linux_libs/scripts_gym/run_test.sh b/.github/unittest/linux_libs/scripts_gym/run_test.sh index ff2f298c9d2..c329871bcbb 100755 --- a/.github/unittest/linux_libs/scripts_gym/run_test.sh +++ b/.github/unittest/linux_libs/scripts_gym/run_test.sh @@ -2,8 +2,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" export PYTORCH_TEST_WITH_SLOW='1' export LAZY_LEGACY_OP=False diff --git a/.github/unittest/linux_libs/scripts_gym/setup_env.sh b/.github/unittest/linux_libs/scripts_gym/setup_env.sh index aade606ba16..558fd4fee61 100755 --- a/.github/unittest/linux_libs/scripts_gym/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_gym/setup_env.sh @@ -9,40 +9,37 @@ set -e this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # Avoid error: "fatal: unsafe repository" -apt-get update && apt-get install -y git wget gcc g++ +apt-get update && apt-get install -y git wget curl gcc g++ cmake apt-get install -y libglfw3 libgl1-mesa-glx libosmesa6 libglew-dev libsdl2-dev libsdl2-2.0-0 apt-get install -y libglvnd0 libgl1 libglx0 libegl1 libgles2 xvfb libegl-dev libx11-dev freeglut3-dev git config --global --add safe.directory '*' root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" -lib_dir="${env_dir}/lib" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac - -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" - -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" - -## 3. Install mujoco + +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" + +# 3. Install build dependencies FIRST (required for C++ extensions AND mujoco-py) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + +## 4. Install mujoco #printf "* Installing mujoco and related\n" #mkdir -p $root_dir/.mujoco #cd $root_dir/.mujoco/ @@ -61,7 +58,9 @@ mkdir -p mujoco_py/binaries/linux \ && rm mujoco.tar.gz wget https://pytorch.s3.amazonaws.com/torchrl/github-artifacts/mjkey.txt cp mjkey.txt mujoco_py/binaries/ -pip install -e . +# Install poetry for mujoco-py build (it uses poetry as build backend) +uv pip install poetry +uv pip install -e . --no-build-isolation cd .. #cd $this_dir @@ -73,9 +72,7 @@ cat "${this_dir}/environment.yml" export MUJOCO_GL=egl -conda env config vars set \ - MAX_IDLE_COUNT=1000 \ - MUJOCO_GL=egl \ +export MAX_IDLE_COUNT=1000 MUJOCO_GL=egl \ SDL_VIDEODRIVER=dummy \ DISPLAY=:99 \ PYOPENGL_PLATFORM=egl \ @@ -88,11 +85,10 @@ conda env config vars set \ # LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/circleci/project/mujoco-py/mujoco_py/binaries/linux/mujoco210/bin # make env variables apparent -conda deactivate && conda activate "${env_dir}" -# pip install pip --upgrade +# uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) #conda install -c conda-forge fltk -y # ROM licence for Atari diff --git a/.github/unittest/linux_libs/scripts_habitat/environment.yml b/.github/unittest/linux_libs/scripts_habitat/environment.yml deleted file mode 100644 index 5a3b65e6aa4..00000000000 --- a/.github/unittest/linux_libs/scripts_habitat/environment.yml +++ /dev/null @@ -1,22 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-error-for-skips - - pytest-rerunfailures - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy==1.9.1 - - hydra-core - - ninja diff --git a/.github/unittest/linux_libs/scripts_habitat/install.sh b/.github/unittest/linux_libs/scripts_habitat/install.sh old mode 100755 new mode 100644 index fcffc5ef55b..185fe7c47e4 --- a/.github/unittest/linux_libs/scripts_habitat/install.sh +++ b/.github/unittest/linux_libs/scripts_habitat/install.sh @@ -5,8 +5,16 @@ unset PYTORCH_VERSION set -e set -v -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +# Make uv available (installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" + +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [[ ${#CU_VERSION} -eq 4 ]]; then CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" @@ -21,24 +29,24 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with %s\n" "${CU_VERSION}" if [[ "$TORCH_VERSION" == "nightly" ]]; then - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U elif [[ "$TORCH_VERSION" == "stable" ]]; then - pip3 install torch --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi -# install tensordict # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test -python3 -c "import functorch;import tensordict" +python -c "import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation # smoke test -python3 -c "import torchrl" + -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_habitat/post_process.sh b/.github/unittest/linux_libs/scripts_habitat/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_habitat/post_process.sh +++ b/.github/unittest/linux_libs/scripts_habitat/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_habitat/requirements.txt b/.github/unittest/linux_libs/scripts_habitat/requirements.txt new file mode 100644 index 00000000000..250c26c4d08 --- /dev/null +++ b/.github/unittest/linux_libs/scripts_habitat/requirements.txt @@ -0,0 +1,17 @@ +pip +hypothesis +future +cloudpickle +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-error-for-skips +pytest-rerunfailures +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy==1.9.1 +hydra-core +ninja diff --git a/.github/unittest/linux_libs/scripts_habitat/run_test.sh b/.github/unittest/linux_libs/scripts_habitat/run_test.sh index 09d1b87f642..750eb8c68a9 100755 --- a/.github/unittest/linux_libs/scripts_habitat/run_test.sh +++ b/.github/unittest/linux_libs/scripts_habitat/run_test.sh @@ -3,28 +3,32 @@ set -e set -v -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" # we can install this now but not before installing tensordict and torchrl, g++ version will break the compilation # https://stackoverflow.com/questions/72540359/glibcxx-3-4-30-not-found-for-librosa-in-conda-virtual-environment-after-tryin -#conda install -y -c conda-forge gcc=12.1.0 -conda install -y -c conda-forge libstdcxx-ng=12 -conda env config vars set \ - MAX_IDLE_COUNT=1000 \ - LD_PRELOAD=$LD_PRELOAD:$STDC_LOC TOKENIZERS_PARALLELISM=true +# Install libstdc++ if needed via apt +apt-get install -y libstdc++6 ## find libstdc -STDC_LOC=$(find conda/ -name "libstdc++.so.6" | head -1) +STDC_LOC=$(find ${root_dir}/.venv/ -name "libstdc++.so.6" 2>/dev/null | head -1 || echo "/usr/lib/x86_64-linux-gnu/libstdc++.so.6") +export MAX_IDLE_COUNT=1000 +export LD_PRELOAD=$LD_PRELOAD:$STDC_LOC +export TOKENIZERS_PARALLELISM=true export PYTORCH_TEST_WITH_SLOW='1' export LAZY_LEGACY_OP=False +export MAGNUM_LOG=quiet +export HABITAT_SIM_LOG=quiet +export MKL_THREADING_LAYER=GNU + python -m torch.utils.collect_env # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -root_dir="$(git rev-parse --show-toplevel)" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" lib_dir="${env_dir}/lib" # smoke test @@ -32,17 +36,6 @@ python -c "import habitat;import habitat.gym" # solves ImportError: /lib64/libstdc++.so.6: version `GLIBCXX_3.4.21' not found export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$lib_dir -export MKL_THREADING_LAYER=GNU -# more logging - -#wget https://github.com/openai/mujoco-py/blob/master/vendor/10_nvidia.json -#mv 10_nvidia.json /usr/share/glvnd/egl_vendor.d/10_nvidia.json - -conda env config vars set \ - MAX_IDLE_COUNT=1000 \ - MAGNUM_LOG=quiet HABITAT_SIM_LOG=quiet TOKENIZERS_PARALLELISM=true - -conda deactivate && conda activate ./env # this workflow only tests the libs diff --git a/.github/unittest/linux_libs/scripts_habitat/setup_env.sh b/.github/unittest/linux_libs/scripts_habitat/setup_env.sh index e8b9e714e9c..3c1ff5a4dd7 100755 --- a/.github/unittest/linux_libs/scripts_habitat/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_habitat/setup_env.sh @@ -10,44 +10,32 @@ set -v this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # Avoid error: "fatal: unsafe repository" -#apt-get update && apt-get install -y git wget gcc g++ unzip +apt-get update && apt-get install -y git wget curl gcc g++ cmake unzip git config --global --add safe.directory '*' root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac +# 1. Install uv +printf "* Installing uv\n" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" - -# 2. Create test environment at ./env +# 2. Create test environment at ./.venv printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" +printf "* Creating a test environment with uv\n" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" # set debug variables -conda env config vars set \ - MAX_IDLE_COUNT=1000 \ - MAGNUM_LOG=debug HABITAT_SIM_LOG=debug TOKENIZERS_PARALLELISM=true -conda deactivate && conda activate "${env_dir}" +export MAX_IDLE_COUNT=1000 +export MAGNUM_LOG=debug +export HABITAT_SIM_LOG=debug +export TOKENIZERS_PARALLELISM=true -pip3 install "cython<3" -conda install -c anaconda cython="<3.0.0" -y +uv pip install "cython<3" # 3. Install git LFS @@ -60,18 +48,20 @@ chmod 755 install.sh cd .. git lfs install -# 4. Install Conda dependencies -printf "* Installing dependencies (except PyTorch)\n" -echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" -cat "${this_dir}/environment.yml" +# 4. Install dependencies +# 3. Install build dependencies FIRST (required for C++ extensions) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" -pip install pip --upgrade +# 4. Install test dependencies and libraries (except PyTorch) +printf "* Installing dependencies from requirements.txt\n" +uv pip install -r "${this_dir}/requirements.txt" + pygame -conda env update --file "${this_dir}/environment.yml" --prune +# Install habitat-sim using pip (conda package not available with uv) +uv pip install habitat-sim -conda install habitat-sim withbullet headless -c conda-forge -c aihabitat -y git clone https://github.com/facebookresearch/habitat-lab.git cd habitat-lab -pip3 install -e habitat-lab -pip3 install -e habitat-baselines # install habitat_baselines -conda run python -m pip install "gym[atari,accept-rom-license]" pygame +uv pip install -e habitat-lab +uv pip install -e habitat-baselines # install habitat_baselines diff --git a/.github/unittest/linux_libs/scripts_isaaclab/isaac.sh b/.github/unittest/linux_libs/scripts_isaaclab/isaac.sh old mode 100755 new mode 100644 index 1068d75f68c..28f69f913e2 --- a/.github/unittest/linux_libs/scripts_isaaclab/isaac.sh +++ b/.github/unittest/linux_libs/scripts_isaaclab/isaac.sh @@ -24,58 +24,58 @@ export OMNI_KIT_ACCEPT_EULA=yes nvidia-smi # Setup -apt-get update && apt-get install -y git wget gcc g++ +apt-get update && apt-get install -y git wget curl gcc g++ apt-get install -y libglfw3 libgl1-mesa-glx libosmesa6 libglew-dev libsdl2-dev libsdl2-2.0-0 apt-get install -y libglvnd0 libgl1 libglx0 libegl1 libgles2 xvfb libegl-dev libx11-dev freeglut3-dev git config --global --add safe.directory '*' root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" -lib_dir="${env_dir}/lib" +env_dir="${root_dir}/.venv" cd "${root_dir}" -# install conda -printf "* Installing conda\n" -wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" -bash ./miniconda.sh -b -f -p "${conda_dir}" -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# Install uv +printf "* Installing uv\n" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" - -conda create --prefix ${env_dir} python=3.10 -y -conda activate ${env_dir} +# Create virtual environment with uv +printf "* Creating a test environment with uv\n" +uv venv "${env_dir}" --python="3.10" +source "${env_dir}/bin/activate" # Pin pytorch to 2.5.1 for IsaacLab -conda install pytorch==2.5.1 torchvision==0.20.1 pytorch-cuda=12.4 -c pytorch -c nvidia -y +uv pip install torch==2.5.1 torchvision==0.20.1 --index-url https://download.pytorch.org/whl/cu124 -conda run -p ${env_dir} pip install --upgrade pip -conda run -p ${env_dir} pip install 'isaacsim[all,extscache]==4.5.0' --extra-index-url https://pypi.nvidia.com -conda install conda-forge::"cmake>3.22" -y +uv pip install 'isaacsim[all,extscache]==4.5.0' --extra-index-url https://pypi.nvidia.com +uv pip install "cmake>3.22" git clone https://github.com/isaac-sim/IsaacLab.git cd IsaacLab -conda run -p ${env_dir} ./isaaclab.sh --install sb3 +./isaaclab.sh --install sb3 cd ../ # install tensordict if [[ "$RELEASE" == 0 ]]; then - conda install "anaconda::cmake>=3.22" -y - conda run -p ${env_dir} python3 -m pip install "pybind11[global]" - conda run -p ${env_dir} python3 -m pip install git+https://github.com/pytorch/tensordict.git + uv pip install "pybind11[global]" + uv pip install git+https://github.com/pytorch/tensordict.git else - conda run -p ${env_dir} python3 -m pip install tensordict + uv pip install tensordict fi # smoke test -conda run -p ${env_dir} python -c "import tensordict" +python -c "import tensordict" + + +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" printf "* Installing torchrl\n" -conda run -p ${env_dir} python setup.py develop -conda run -p ${env_dir} python -c "import torchrl" +uv pip install -e . --no-build-isolation +python -c "import torchrl" # Install pytest -conda run -p ${env_dir} python -m pip install pytest pytest-cov pytest-mock pytest-instafail pytest-rerunfailures pytest-error-for-skips pytest-asyncio +uv pip install pytest pytest-cov pytest-mock pytest-instafail pytest-rerunfailures pytest-error-for-skips pytest-asyncio # Run tests -conda run -p ${env_dir} python -m pytest test/test_libs.py -k isaac -s +python -m pytest test/test_libs.py -k isaac -s diff --git a/.github/unittest/linux_libs/scripts_jumanji/environment.yml b/.github/unittest/linux_libs/scripts_jumanji/environment.yml deleted file mode 100644 index 014c7956ec2..00000000000 --- a/.github/unittest/linux_libs/scripts_jumanji/environment.yml +++ /dev/null @@ -1,23 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - jumanji - - gymnasium<1.0.0 diff --git a/.github/unittest/linux_libs/scripts_jumanji/install.sh b/.github/unittest/linux_libs/scripts_jumanji/install.sh old mode 100755 new mode 100644 index ce1ba3ff13e..ef95d79d899 --- a/.github/unittest/linux_libs/scripts_jumanji/install.sh +++ b/.github/unittest/linux_libs/scripts_jumanji/install.sh @@ -7,8 +7,15 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +# Add uv to PATH (it was installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -28,15 +35,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu else - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -45,16 +52,17 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import functorch;import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation # smoke test python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_jumanji/post_process.sh b/.github/unittest/linux_libs/scripts_jumanji/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_jumanji/post_process.sh +++ b/.github/unittest/linux_libs/scripts_jumanji/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_jumanji/requirements.txt b/.github/unittest/linux_libs/scripts_jumanji/requirements.txt new file mode 100644 index 00000000000..a6713a74aa2 --- /dev/null +++ b/.github/unittest/linux_libs/scripts_jumanji/requirements.txt @@ -0,0 +1,18 @@ +pip +hypothesis +future +cloudpickle +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +jumanji +gymnasium<1.0.0 diff --git a/.github/unittest/linux_libs/scripts_jumanji/run_test.sh b/.github/unittest/linux_libs/scripts_jumanji/run_test.sh index 5ac898e8a8f..8ca732dfd1d 100755 --- a/.github/unittest/linux_libs/scripts_jumanji/run_test.sh +++ b/.github/unittest/linux_libs/scripts_jumanji/run_test.sh @@ -2,8 +2,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" apt-get update && apt-get install -y git wget cmake diff --git a/.github/unittest/linux_libs/scripts_jumanji/setup_env.sh b/.github/unittest/linux_libs/scripts_jumanji/setup_env.sh index 4e87b2e9ed1..083b2863733 100755 --- a/.github/unittest/linux_libs/scripts_jumanji/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_jumanji/setup_env.sh @@ -9,7 +9,7 @@ set -e apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -32,31 +32,26 @@ apt-get upgrade -y libstdc++6 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac - -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" - -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" + +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" + ## 3. Install mujoco #printf "* Installing mujoco and related\n" @@ -73,9 +68,9 @@ printf "* Installing dependencies (except PyTorch)\n" echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" -pip install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) #yum makecache #yum -y install glfw-devel diff --git a/.github/unittest/linux_libs/scripts_llm/environment.yml b/.github/unittest/linux_libs/scripts_llm/environment.yml deleted file mode 100644 index 2e6b3c6e173..00000000000 --- a/.github/unittest/linux_libs/scripts_llm/environment.yml +++ /dev/null @@ -1,24 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - transformers - - datasets - - vllm diff --git a/.github/unittest/linux_libs/scripts_llm/install.sh b/.github/unittest/linux_libs/scripts_llm/install.sh index 68e63bf58ca..678fdf4f862 100644 --- a/.github/unittest/linux_libs/scripts_llm/install.sh +++ b/.github/unittest/linux_libs/scripts_llm/install.sh @@ -7,8 +7,15 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +# Add uv to PATH (it was installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -30,15 +37,15 @@ git submodule sync && git submodule update --init --recursive #printf "Installing PyTorch with cu128" #if [[ "$TORCH_VERSION" == "nightly" ]]; then # if [ "${CU_VERSION:-}" == cpu ] ; then -# pip3 install --pre torch "numpy<2.0.0" --index-url https://download.pytorch.org/whl/nightly/cpu -U +# uv pip install --pre torch "numpy<2.0.0" --index-url https://download.pytorch.org/whl/nightly/cpu -U # else -# pip3 install --pre torch "numpy<2.0.0" --index-url https://download.pytorch.org/whl/nightly/cu128 -U +# uv pip install --pre torch "numpy<2.0.0" --index-url https://download.pytorch.org/whl/nightly/cu128 -U # fi #elif [[ "$TORCH_VERSION" == "stable" ]]; then # if [ "${CU_VERSION:-}" == cpu ] ; then -# pip3 install torch "numpy<2.0.0" --index-url https://download.pytorch.org/whl/cpu +# uv pip install torch "numpy<2.0.0" --index-url https://download.pytorch.org/whl/cpu # else -# pip3 install torch "numpy<2.0.0" --index-url https://download.pytorch.org/whl/cu128 +# uv pip install torch "numpy<2.0.0" --index-url https://download.pytorch.org/whl/cu128 # fi #else # printf "Failed to install pytorch" @@ -47,16 +54,17 @@ git submodule sync && git submodule update --init --recursive # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation # smoke test python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_llm/post_process.sh b/.github/unittest/linux_libs/scripts_llm/post_process.sh index e97bf2a7b1b..6647ab38833 100644 --- a/.github/unittest/linux_libs/scripts_llm/post_process.sh +++ b/.github/unittest/linux_libs/scripts_llm/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_llm/requirements.txt b/.github/unittest/linux_libs/scripts_llm/requirements.txt new file mode 100644 index 00000000000..e4eb4f4ae86 --- /dev/null +++ b/.github/unittest/linux_libs/scripts_llm/requirements.txt @@ -0,0 +1,19 @@ +pip +hypothesis +future +cloudpickle +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +transformers +datasets +vllm diff --git a/.github/unittest/linux_libs/scripts_llm/run_test.sh b/.github/unittest/linux_libs/scripts_llm/run_test.sh index ac60ae37f1e..6a36df57338 100644 --- a/.github/unittest/linux_libs/scripts_llm/run_test.sh +++ b/.github/unittest/linux_libs/scripts_llm/run_test.sh @@ -2,8 +2,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" apt-get update && apt-get install -y git gcc cmake ln -s /usr/bin/swig3.0 /usr/bin/swig @@ -21,7 +22,7 @@ root_dir="$(git rev-parse --show-toplevel)" env_dir="${root_dir}/env" lib_dir="${env_dir}/lib" -conda deactivate && conda activate ./env +# conda deactivate (not needed with uv) && source ./.venv/bin/activate python -c "import transformers, datasets" diff --git a/.github/unittest/linux_libs/scripts_llm/setup_env.sh b/.github/unittest/linux_libs/scripts_llm/setup_env.sh index 53dfc0bd50b..1754a9dc03a 100644 --- a/.github/unittest/linux_libs/scripts_llm/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_llm/setup_env.sh @@ -9,7 +9,7 @@ set -e apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -31,31 +31,26 @@ apt-get upgrade -y libstdc++6 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" # 3. Install Conda dependencies @@ -63,6 +58,6 @@ printf "* Installing dependencies (except PyTorch)\n" echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" -pip install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) diff --git a/.github/unittest/linux_libs/scripts_meltingpot/environment.yml b/.github/unittest/linux_libs/scripts_meltingpot/environment.yml deleted file mode 100644 index e003419348c..00000000000 --- a/.github/unittest/linux_libs/scripts_meltingpot/environment.yml +++ /dev/null @@ -1,17 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - cloudpickle - - torch - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] diff --git a/.github/unittest/linux_libs/scripts_meltingpot/install.sh b/.github/unittest/linux_libs/scripts_meltingpot/install.sh old mode 100755 new mode 100644 index 189d3dc8412..061b7f9cabe --- a/.github/unittest/linux_libs/scripts_meltingpot/install.sh +++ b/.github/unittest/linux_libs/scripts_meltingpot/install.sh @@ -7,8 +7,16 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +# Make uv available (installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" + +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -28,15 +36,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch --index-url https://download.pytorch.org/whl/cpu + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu else - pip3 install torch --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -45,23 +53,25 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation python -c "import torchrl" -conda install conda-forge::jq -y +# Install jq using apt +apt-get install -y jq # Install meltingpot from git -#pip3 install dmlab2d +#uv pip install dmlab2d LATEST_TAG=$(curl "https://api.github.com/repos/google-deepmind/meltingpot/tags" | jq -r '.[0].name') echo $(ldd --version) -pip3 install git+https://github.com/google-deepmind/meltingpot@${LATEST_TAG} +uv pip install git+https://github.com/google-deepmind/meltingpot@${LATEST_TAG} diff --git a/.github/unittest/linux_libs/scripts_meltingpot/post_process.sh b/.github/unittest/linux_libs/scripts_meltingpot/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_meltingpot/post_process.sh +++ b/.github/unittest/linux_libs/scripts_meltingpot/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_meltingpot/requirements.txt b/.github/unittest/linux_libs/scripts_meltingpot/requirements.txt new file mode 100644 index 00000000000..dbf84ff3cb4 --- /dev/null +++ b/.github/unittest/linux_libs/scripts_meltingpot/requirements.txt @@ -0,0 +1,12 @@ +pip +cloudpickle +torch +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] diff --git a/.github/unittest/linux_libs/scripts_meltingpot/run_test.sh b/.github/unittest/linux_libs/scripts_meltingpot/run_test.sh index 0636639a613..dc09e472c42 100755 --- a/.github/unittest/linux_libs/scripts_meltingpot/run_test.sh +++ b/.github/unittest/linux_libs/scripts_meltingpot/run_test.sh @@ -2,8 +2,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" apt-get update && apt-get install -y git wget cmake diff --git a/.github/unittest/linux_libs/scripts_meltingpot/setup_env.sh b/.github/unittest/linux_libs/scripts_meltingpot/setup_env.sh index 305659eb26b..e9886389068 100755 --- a/.github/unittest/linux_libs/scripts_meltingpot/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_meltingpot/setup_env.sh @@ -9,7 +9,7 @@ set -e -v apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -33,38 +33,33 @@ this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" # 4. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" -pip install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) diff --git a/.github/unittest/linux_libs/scripts_minari/environment.yml b/.github/unittest/linux_libs/scripts_minari/environment.yml deleted file mode 100644 index 99d911ca71a..00000000000 --- a/.github/unittest/linux_libs/scripts_minari/environment.yml +++ /dev/null @@ -1,30 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - minari[gcs,hdf5,hf] - - gymnasium>=1.2.0 - - ale-py - - gymnasium-robotics - - minari[create] - - jax>=0.7.0 - - mujoco<3.3.6 - - mujoco-py<2.2,>=2.1 - - minigrid diff --git a/.github/unittest/linux_libs/scripts_minari/install.sh b/.github/unittest/linux_libs/scripts_minari/install.sh old mode 100755 new mode 100644 index 0fc3e9076db..9fb20d38c18 --- a/.github/unittest/linux_libs/scripts_minari/install.sh +++ b/.github/unittest/linux_libs/scripts_minari/install.sh @@ -7,8 +7,15 @@ unset PYTORCH_VERSION set -e +# Make uv available (installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" + # Note: This script is sourced by run_all.sh, so the environment is already active +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" else @@ -27,15 +34,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - uv pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - uv pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - uv pip install torch --index-url https://download.pytorch.org/whl/cpu + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu else - uv pip install torch --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -52,8 +59,9 @@ fi # smoke test python -c "import functorch;import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation # smoke test python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_minari/requirements.txt b/.github/unittest/linux_libs/scripts_minari/requirements.txt index ae21314f263..f0638585a27 100644 --- a/.github/unittest/linux_libs/scripts_minari/requirements.txt +++ b/.github/unittest/linux_libs/scripts_minari/requirements.txt @@ -1,3 +1,4 @@ +pip hypothesis future cloudpickle @@ -13,8 +14,12 @@ pybind11[global] pyyaml scipy hydra-core -minari[gcs,hdf5,hf,create] +minari[gcs,hdf5,hf] gymnasium>=1.2.0 ale-py gymnasium-robotics -mujoco +minari[create] +jax>=0.7.0 +mujoco<3.3.6 +mujoco-py<2.2,>=2.1 +minigrid diff --git a/.github/unittest/linux_libs/scripts_minari/run_test.sh b/.github/unittest/linux_libs/scripts_minari/run_test.sh index e75868fb41e..c5917630a61 100755 --- a/.github/unittest/linux_libs/scripts_minari/run_test.sh +++ b/.github/unittest/linux_libs/scripts_minari/run_test.sh @@ -2,6 +2,8 @@ set -e +export PATH="$HOME/.local/bin:$PATH" + # Note: This script is sourced by run_all.sh, so the environment is already active apt-get update && apt-get remove swig -y && apt-get install -y git gcc patchelf libosmesa6-dev libgl1-mesa-glx libglfw3 swig3.0 cmake diff --git a/.github/unittest/linux_libs/scripts_minari/setup_env.sh b/.github/unittest/linux_libs/scripts_minari/setup_env.sh index a7133a9fc47..591c495145a 100755 --- a/.github/unittest/linux_libs/scripts_minari/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_minari/setup_env.sh @@ -11,7 +11,7 @@ set -v apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -41,9 +41,7 @@ cd "${root_dir}" if ! command -v uv &> /dev/null; then printf "* Installing uv\n" # Try different Python commands - if command -v python3 &> /dev/null; then - python3 -m pip install uv - elif command -v python &> /dev/null; then + if command -v python &> /dev/null; then python -m pip install uv else # Fallback to curl installation diff --git a/.github/unittest/linux_libs/scripts_open_spiel/environment.yml b/.github/unittest/linux_libs/scripts_open_spiel/environment.yml deleted file mode 100644 index 0ed33d56f31..00000000000 --- a/.github/unittest/linux_libs/scripts_open_spiel/environment.yml +++ /dev/null @@ -1,21 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - open_spiel diff --git a/.github/unittest/linux_libs/scripts_open_spiel/install.sh b/.github/unittest/linux_libs/scripts_open_spiel/install.sh old mode 100755 new mode 100644 index 7ed6498d2c0..ef95d79d899 --- a/.github/unittest/linux_libs/scripts_open_spiel/install.sh +++ b/.github/unittest/linux_libs/scripts_open_spiel/install.sh @@ -7,8 +7,15 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +# Add uv to PATH (it was installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -28,15 +35,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch --index-url https://download.pytorch.org/whl/cpu + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu else - pip3 install torch --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -45,16 +52,17 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import functorch;import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation # smoke test python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_open_spiel/post_process.sh b/.github/unittest/linux_libs/scripts_open_spiel/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_open_spiel/post_process.sh +++ b/.github/unittest/linux_libs/scripts_open_spiel/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_open_spiel/requirements.txt b/.github/unittest/linux_libs/scripts_open_spiel/requirements.txt new file mode 100644 index 00000000000..33573a2c8fa --- /dev/null +++ b/.github/unittest/linux_libs/scripts_open_spiel/requirements.txt @@ -0,0 +1,16 @@ +pip +hypothesis +future +cloudpickle +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +open_spiel diff --git a/.github/unittest/linux_libs/scripts_open_spiel/run_test.sh b/.github/unittest/linux_libs/scripts_open_spiel/run_test.sh index 98952d1d4b0..f7de77a36ac 100755 --- a/.github/unittest/linux_libs/scripts_open_spiel/run_test.sh +++ b/.github/unittest/linux_libs/scripts_open_spiel/run_test.sh @@ -2,8 +2,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" apt-get update && apt-get install -y git wget cmake @@ -17,7 +18,7 @@ root_dir="$(git rev-parse --show-toplevel)" env_dir="${root_dir}/env" lib_dir="${env_dir}/lib" -conda deactivate && conda activate ./env +# conda deactivate (not needed with uv) && source ./.venv/bin/activate # this workflow only tests the libs python -c "import pyspiel" diff --git a/.github/unittest/linux_libs/scripts_open_spiel/setup_env.sh b/.github/unittest/linux_libs/scripts_open_spiel/setup_env.sh index 19c821637de..90129e2f240 100755 --- a/.github/unittest/linux_libs/scripts_open_spiel/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_open_spiel/setup_env.sh @@ -11,7 +11,7 @@ set -v apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -33,37 +33,32 @@ apt-get upgrade -y libstdc++6 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" # 3. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" -pip install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) diff --git a/.github/unittest/linux_libs/scripts_openx/environment.yml b/.github/unittest/linux_libs/scripts_openx/environment.yml deleted file mode 100644 index 353f3de438e..00000000000 --- a/.github/unittest/linux_libs/scripts_openx/environment.yml +++ /dev/null @@ -1,25 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - tqdm - - h5py - - datasets<4.0.0 - - pillow diff --git a/.github/unittest/linux_libs/scripts_openx/install.sh b/.github/unittest/linux_libs/scripts_openx/install.sh old mode 100755 new mode 100644 index 6a3532d6ef0..fbac3bc6188 --- a/.github/unittest/linux_libs/scripts_openx/install.sh +++ b/.github/unittest/linux_libs/scripts_openx/install.sh @@ -7,8 +7,15 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +# Add uv to PATH (it was installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -29,15 +36,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu -U + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu -U else - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cu128 -U + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 -U fi else printf "Failed to install pytorch" @@ -46,16 +53,17 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import functorch;import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation # smoke test python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_openx/post_process.sh b/.github/unittest/linux_libs/scripts_openx/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_openx/post_process.sh +++ b/.github/unittest/linux_libs/scripts_openx/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_openx/requirements.txt b/.github/unittest/linux_libs/scripts_openx/requirements.txt new file mode 100644 index 00000000000..b56802639f3 --- /dev/null +++ b/.github/unittest/linux_libs/scripts_openx/requirements.txt @@ -0,0 +1,20 @@ +pip +hypothesis +future +cloudpickle +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +tqdm +h5py +datasets<4.0.0 +pillow diff --git a/.github/unittest/linux_libs/scripts_openx/run_test.sh b/.github/unittest/linux_libs/scripts_openx/run_test.sh index 2eaf66553bd..513185ce16d 100755 --- a/.github/unittest/linux_libs/scripts_openx/run_test.sh +++ b/.github/unittest/linux_libs/scripts_openx/run_test.sh @@ -2,8 +2,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" apt-get update && apt-get remove swig -y && apt-get install -y git gcc patchelf libosmesa6-dev libgl1-mesa-glx libglfw3 swig3.0 cmake ln -s /usr/bin/swig3.0 /usr/bin/swig @@ -18,7 +19,7 @@ root_dir="$(git rev-parse --show-toplevel)" env_dir="${root_dir}/env" lib_dir="${env_dir}/lib" -conda deactivate && conda activate ./env +# conda deactivate (not needed with uv) && source ./.venv/bin/activate python .github/unittest/helpers/coverage_run_parallel.py -m pytest test/test_libs.py --instafail -v --durations 200 --capture no -k TestOpenX --error-for-skips --runslow coverage combine diff --git a/.github/unittest/linux_libs/scripts_openx/setup_env.sh b/.github/unittest/linux_libs/scripts_openx/setup_env.sh index a7afd646646..90129e2f240 100755 --- a/.github/unittest/linux_libs/scripts_openx/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_openx/setup_env.sh @@ -11,7 +11,7 @@ set -v apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -33,37 +33,32 @@ apt-get upgrade -y libstdc++6 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" # 3. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" -pip3 install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) diff --git a/.github/unittest/linux_libs/scripts_pettingzoo/environment.yml b/.github/unittest/linux_libs/scripts_pettingzoo/environment.yml deleted file mode 100644 index 92edaf1aa7a..00000000000 --- a/.github/unittest/linux_libs/scripts_pettingzoo/environment.yml +++ /dev/null @@ -1,24 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - swig - - pip: - - cloudpickle - - gym - - gym-notices - - six - - zipp - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - expecttest - - pybind11[global] - - pyyaml - - autorom[accept-rom-license] - - pettingzoo[all]==1.24.3 - - gymnasium<1.0.0 diff --git a/.github/unittest/linux_libs/scripts_pettingzoo/install.sh b/.github/unittest/linux_libs/scripts_pettingzoo/install.sh old mode 100755 new mode 100644 index 137a6cb827a..b7aa004e6d6 --- a/.github/unittest/linux_libs/scripts_pettingzoo/install.sh +++ b/.github/unittest/linux_libs/scripts_pettingzoo/install.sh @@ -7,8 +7,15 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +# Add uv to PATH (it was installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -28,15 +35,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch --index-url https://download.pytorch.org/whl/cpu + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu else - pip3 install torch --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -45,14 +52,15 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_pettingzoo/post_process.sh b/.github/unittest/linux_libs/scripts_pettingzoo/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_pettingzoo/post_process.sh +++ b/.github/unittest/linux_libs/scripts_pettingzoo/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_pettingzoo/requirements.txt b/.github/unittest/linux_libs/scripts_pettingzoo/requirements.txt new file mode 100644 index 00000000000..189090d8d3b --- /dev/null +++ b/.github/unittest/linux_libs/scripts_pettingzoo/requirements.txt @@ -0,0 +1,19 @@ +pip +swig +cloudpickle +gym +gym-notices +six +zipp +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +expecttest +pybind11[global] +pyyaml +autorom[accept-rom-license] +pettingzoo[all]==1.24.3 +gymnasium<1.0.0 diff --git a/.github/unittest/linux_libs/scripts_pettingzoo/run_test.sh b/.github/unittest/linux_libs/scripts_pettingzoo/run_test.sh index 5e4ea0ec31b..410da60e4d1 100755 --- a/.github/unittest/linux_libs/scripts_pettingzoo/run_test.sh +++ b/.github/unittest/linux_libs/scripts_pettingzoo/run_test.sh @@ -2,8 +2,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" apt-get update && apt-get install -y git wget cmake diff --git a/.github/unittest/linux_libs/scripts_pettingzoo/setup_env.sh b/.github/unittest/linux_libs/scripts_pettingzoo/setup_env.sh index 75c4d4e861a..6698edc4d6f 100755 --- a/.github/unittest/linux_libs/scripts_pettingzoo/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_pettingzoo/setup_env.sh @@ -10,7 +10,7 @@ set -e apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -32,40 +32,35 @@ apt-get upgrade -y libstdc++6 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" # 4. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" -pip install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) # 5. Download atari roms autorom_dir="${env_dir}/lib/python${PYTHON_VERSION}/site-packages/AutoROM/roms" diff --git a/.github/unittest/linux_libs/scripts_robohive/environment.yml b/.github/unittest/linux_libs/scripts_robohive/environment.yml deleted file mode 100644 index a667b841525..00000000000 --- a/.github/unittest/linux_libs/scripts_robohive/environment.yml +++ /dev/null @@ -1,31 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - protobuf - - pip: - # Initial version is required to install Atari ROMS in setup_env.sh - - gymnasium<1.0 - - hypothesis - - future - - cloudpickle - - pygame - - moviepy<2.0.0 - - tqdm - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - patchelf - - mujoco==2.3.3 - - dm_control==1.0.11 - - numpy<2.0.0 diff --git a/.github/unittest/linux_libs/scripts_robohive/install_and_run_test.sh b/.github/unittest/linux_libs/scripts_robohive/install_and_run_test.sh old mode 100755 new mode 100644 index b5078acee7f..3ca9b6267d6 --- a/.github/unittest/linux_libs/scripts_robohive/install_and_run_test.sh +++ b/.github/unittest/linux_libs/scripts_robohive/install_and_run_test.sh @@ -9,8 +9,13 @@ apt-get update && apt-get install -y git wget libglew-dev libx11-dev x11proto-de set -e set -v -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then cudatoolkit="cpuonly" @@ -37,15 +42,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch --index-url https://download.pytorch.org/whl/cpu + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu else - pip3 install torch --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -54,16 +59,17 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation python -c "import torchrl" # Extracted from run_test.sh to run once. diff --git a/.github/unittest/linux_libs/scripts_robohive/post_process.sh b/.github/unittest/linux_libs/scripts_robohive/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_robohive/post_process.sh +++ b/.github/unittest/linux_libs/scripts_robohive/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_robohive/requirements.txt b/.github/unittest/linux_libs/scripts_robohive/requirements.txt new file mode 100644 index 00000000000..8aa09c69aef --- /dev/null +++ b/.github/unittest/linux_libs/scripts_robohive/requirements.txt @@ -0,0 +1,25 @@ +pip +protobuf +gymnasium<1.0 +hypothesis +future +cloudpickle +pygame +moviepy<2.0.0 +tqdm +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +patchelf +mujoco==2.3.3 +dm_control==1.0.11 +numpy<2.0.0 diff --git a/.github/unittest/linux_libs/scripts_robohive/setup_env.sh b/.github/unittest/linux_libs/scripts_robohive/setup_env.sh index 44dd16585fd..bbe5e96248f 100755 --- a/.github/unittest/linux_libs/scripts_robohive/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_robohive/setup_env.sh @@ -9,36 +9,30 @@ set -e this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # Avoid error: "fatal: unsafe repository" -apt-get update && apt-get install -y git wget gcc g++ cmake +apt-get update && apt-get install -y git wget curl gcc g++ cmake git config --global --add safe.directory '*' root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" -lib_dir="${env_dir}/lib" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac - -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" - -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" + +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" + #git clone https://github.com/vmoens/mujoco-py.git #cd mujoco-py @@ -49,7 +43,7 @@ conda activate "${env_dir}" # && rm mujoco.tar.gz #wget https://pytorch.s3.amazonaws.com/torchrl/github-artifacts/mjkey.txt #cp mjkey.txt mujoco_py/binaries/ -#pip install -e . +#uv pip install -e . --no-build-isolation #cd .. #cd $this_dir @@ -60,9 +54,7 @@ echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" export MUJOCO_GL=egl -conda env config vars set \ - MAX_IDLE_COUNT=1000 \ - MUJOCO_GL=egl \ +export MAX_IDLE_COUNT=1000 MUJOCO_GL=egl \ SDL_VIDEODRIVER=dummy \ DISPLAY=:99 \ PYOPENGL_PLATFORM=egl \ @@ -72,17 +64,16 @@ conda env config vars set \ TOKENIZERS_PARALLELISM=true # make env variables apparent -conda deactivate && conda activate "${env_dir}" -pip install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) -conda install conda-forge::ffmpeg -y +apt-get install -y ffmpeg -pip install robohive +uv pip install robohive -python3 -m robohive_init +python -m robohive_init # make sure only gymnasium is available # pip uninstall gym -y diff --git a/.github/unittest/linux_libs/scripts_roboset/environment.yml b/.github/unittest/linux_libs/scripts_roboset/environment.yml deleted file mode 100644 index 386ec24b3c2..00000000000 --- a/.github/unittest/linux_libs/scripts_roboset/environment.yml +++ /dev/null @@ -1,24 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - huggingface_hub - - tqdm - - h5py diff --git a/.github/unittest/linux_libs/scripts_roboset/install.sh b/.github/unittest/linux_libs/scripts_roboset/install.sh old mode 100755 new mode 100644 index 0094f045932..c43fab106c6 --- a/.github/unittest/linux_libs/scripts_roboset/install.sh +++ b/.github/unittest/linux_libs/scripts_roboset/install.sh @@ -7,8 +7,15 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +# Add uv to PATH (it was installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -29,15 +36,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch --index-url https://download.pytorch.org/whl/cpu + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu else - pip3 install torch --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -46,16 +53,17 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import functorch;import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation # smoke test python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_roboset/post_process.sh b/.github/unittest/linux_libs/scripts_roboset/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_roboset/post_process.sh +++ b/.github/unittest/linux_libs/scripts_roboset/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_roboset/requirements.txt b/.github/unittest/linux_libs/scripts_roboset/requirements.txt new file mode 100644 index 00000000000..27aee99abf5 --- /dev/null +++ b/.github/unittest/linux_libs/scripts_roboset/requirements.txt @@ -0,0 +1,19 @@ +pip +hypothesis +future +cloudpickle +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +huggingface_hub +tqdm +h5py diff --git a/.github/unittest/linux_libs/scripts_roboset/run_test.sh b/.github/unittest/linux_libs/scripts_roboset/run_test.sh index 1203390a1c9..8328d1bff9f 100755 --- a/.github/unittest/linux_libs/scripts_roboset/run_test.sh +++ b/.github/unittest/linux_libs/scripts_roboset/run_test.sh @@ -2,8 +2,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" apt-get update && apt-get remove swig -y && apt-get install -y git gcc patchelf libosmesa6-dev libgl1-mesa-glx libglfw3 swig3.0 cmake ln -s /usr/bin/swig3.0 /usr/bin/swig @@ -18,7 +19,7 @@ root_dir="$(git rev-parse --show-toplevel)" env_dir="${root_dir}/env" lib_dir="${env_dir}/lib" -conda deactivate && conda activate ./env +# conda deactivate (not needed with uv) && source ./.venv/bin/activate python .github/unittest/helpers/coverage_run_parallel.py -m pytest test/test_libs.py --instafail -v --durations 200 --capture no -k TestRoboset --error-for-skips --runslow coverage combine diff --git a/.github/unittest/linux_libs/scripts_roboset/setup_env.sh b/.github/unittest/linux_libs/scripts_roboset/setup_env.sh index a7afd646646..90129e2f240 100755 --- a/.github/unittest/linux_libs/scripts_roboset/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_roboset/setup_env.sh @@ -11,7 +11,7 @@ set -v apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -33,37 +33,32 @@ apt-get upgrade -y libstdc++6 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" # 3. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" -pip3 install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) diff --git a/.github/unittest/linux_libs/scripts_sklearn/environment.yml b/.github/unittest/linux_libs/scripts_sklearn/environment.yml deleted file mode 100644 index 251f66c8765..00000000000 --- a/.github/unittest/linux_libs/scripts_sklearn/environment.yml +++ /dev/null @@ -1,23 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - scikit-learn - - pandas diff --git a/.github/unittest/linux_libs/scripts_sklearn/install.sh b/.github/unittest/linux_libs/scripts_sklearn/install.sh old mode 100755 new mode 100644 index 0094f045932..c43fab106c6 --- a/.github/unittest/linux_libs/scripts_sklearn/install.sh +++ b/.github/unittest/linux_libs/scripts_sklearn/install.sh @@ -7,8 +7,15 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +# Add uv to PATH (it was installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -29,15 +36,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch --index-url https://download.pytorch.org/whl/cpu + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu else - pip3 install torch --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -46,16 +53,17 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import functorch;import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation # smoke test python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_sklearn/post_process.sh b/.github/unittest/linux_libs/scripts_sklearn/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_sklearn/post_process.sh +++ b/.github/unittest/linux_libs/scripts_sklearn/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_sklearn/requirements.txt b/.github/unittest/linux_libs/scripts_sklearn/requirements.txt new file mode 100644 index 00000000000..595dd77a552 --- /dev/null +++ b/.github/unittest/linux_libs/scripts_sklearn/requirements.txt @@ -0,0 +1,18 @@ +pip +hypothesis +future +cloudpickle +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +scikit-learn +pandas diff --git a/.github/unittest/linux_libs/scripts_sklearn/run_test.sh b/.github/unittest/linux_libs/scripts_sklearn/run_test.sh index ed946e36d60..549f8c85999 100755 --- a/.github/unittest/linux_libs/scripts_sklearn/run_test.sh +++ b/.github/unittest/linux_libs/scripts_sklearn/run_test.sh @@ -2,8 +2,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" apt-get update && apt-get install -y git gcc cmake ln -s /usr/bin/swig3.0 /usr/bin/swig @@ -18,7 +19,7 @@ root_dir="$(git rev-parse --show-toplevel)" env_dir="${root_dir}/env" lib_dir="${env_dir}/lib" -conda deactivate && conda activate ./env +# conda deactivate (not needed with uv) && source ./.venv/bin/activate # this workflow only tests the libs python -c "import sklearn, pandas" diff --git a/.github/unittest/linux_libs/scripts_sklearn/setup_env.sh b/.github/unittest/linux_libs/scripts_sklearn/setup_env.sh index ac629308266..6914c841fae 100755 --- a/.github/unittest/linux_libs/scripts_sklearn/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_sklearn/setup_env.sh @@ -10,7 +10,7 @@ set -e apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -32,31 +32,26 @@ apt-get upgrade -y libstdc++6 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" # 3. Install Conda dependencies @@ -64,6 +59,6 @@ printf "* Installing dependencies (except PyTorch)\n" echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" -pip install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) diff --git a/.github/unittest/linux_libs/scripts_smacv2/environment.yml b/.github/unittest/linux_libs/scripts_smacv2/environment.yml deleted file mode 100644 index 45d9815885f..00000000000 --- a/.github/unittest/linux_libs/scripts_smacv2/environment.yml +++ /dev/null @@ -1,22 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - cloudpickle - - gym - - gym-notices - - zipp - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - numpy==1.23.0 - - git+https://github.com/oxwhirl/smacv2.git diff --git a/.github/unittest/linux_libs/scripts_smacv2/install.sh b/.github/unittest/linux_libs/scripts_smacv2/install.sh old mode 100755 new mode 100644 index 137a6cb827a..b7aa004e6d6 --- a/.github/unittest/linux_libs/scripts_smacv2/install.sh +++ b/.github/unittest/linux_libs/scripts_smacv2/install.sh @@ -7,8 +7,15 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +# Add uv to PATH (it was installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -28,15 +35,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch --index-url https://download.pytorch.org/whl/cpu + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu else - pip3 install torch --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -45,14 +52,15 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_smacv2/post_process.sh b/.github/unittest/linux_libs/scripts_smacv2/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_smacv2/post_process.sh +++ b/.github/unittest/linux_libs/scripts_smacv2/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_smacv2/requirements.txt b/.github/unittest/linux_libs/scripts_smacv2/requirements.txt new file mode 100644 index 00000000000..5b29003bfe0 --- /dev/null +++ b/.github/unittest/linux_libs/scripts_smacv2/requirements.txt @@ -0,0 +1,17 @@ +pip +cloudpickle +gym +gym-notices +zipp +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] +pyyaml +numpy==1.23.0 +git+https://github.com/oxwhirl/smacv2.git diff --git a/.github/unittest/linux_libs/scripts_smacv2/run_test.sh b/.github/unittest/linux_libs/scripts_smacv2/run_test.sh index 7d063305dc0..077e7af7fac 100755 --- a/.github/unittest/linux_libs/scripts_smacv2/run_test.sh +++ b/.github/unittest/linux_libs/scripts_smacv2/run_test.sh @@ -2,8 +2,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" apt-get update && apt-get install -y git wget cmake diff --git a/.github/unittest/linux_libs/scripts_smacv2/setup_env.sh b/.github/unittest/linux_libs/scripts_smacv2/setup_env.sh index ca18a0342c2..0f486f1fd86 100755 --- a/.github/unittest/linux_libs/scripts_smacv2/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_smacv2/setup_env.sh @@ -11,7 +11,7 @@ this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -36,40 +36,35 @@ ln -s libglut.so.3.12 libglut.so.3 cd $this_dir root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac - -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" - -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" + +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" + # 4. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" -pip install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) # 5. Install StarCraft 2 with SMACv2 maps starcraft_path="${root_dir}/StarCraftII" diff --git a/.github/unittest/linux_libs/scripts_unity_mlagents/environment.yml b/.github/unittest/linux_libs/scripts_unity_mlagents/environment.yml deleted file mode 100644 index 4bfe74674eb..00000000000 --- a/.github/unittest/linux_libs/scripts_unity_mlagents/environment.yml +++ /dev/null @@ -1,22 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - python==3.10.12 - - pip - - pip: - - mlagents_envs==1.0.0 - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core diff --git a/.github/unittest/linux_libs/scripts_unity_mlagents/install.sh b/.github/unittest/linux_libs/scripts_unity_mlagents/install.sh old mode 100755 new mode 100644 index 7ed6498d2c0..ef95d79d899 --- a/.github/unittest/linux_libs/scripts_unity_mlagents/install.sh +++ b/.github/unittest/linux_libs/scripts_unity_mlagents/install.sh @@ -7,8 +7,15 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +# Add uv to PATH (it was installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -28,15 +35,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch --index-url https://download.pytorch.org/whl/cpu + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu else - pip3 install torch --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -45,16 +52,17 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import functorch;import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation # smoke test python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_unity_mlagents/post_process.sh b/.github/unittest/linux_libs/scripts_unity_mlagents/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_unity_mlagents/post_process.sh +++ b/.github/unittest/linux_libs/scripts_unity_mlagents/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_unity_mlagents/requirements.txt b/.github/unittest/linux_libs/scripts_unity_mlagents/requirements.txt new file mode 100644 index 00000000000..07724138c2b --- /dev/null +++ b/.github/unittest/linux_libs/scripts_unity_mlagents/requirements.txt @@ -0,0 +1,16 @@ +pip +mlagents_envs==1.0.0 +hypothesis +future +cloudpickle +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +expecttest +pybind11[global] +pyyaml +scipy +hydra-core diff --git a/.github/unittest/linux_libs/scripts_unity_mlagents/run_test.sh b/.github/unittest/linux_libs/scripts_unity_mlagents/run_test.sh index 02c8a36b5be..0dc44ad8ea2 100755 --- a/.github/unittest/linux_libs/scripts_unity_mlagents/run_test.sh +++ b/.github/unittest/linux_libs/scripts_unity_mlagents/run_test.sh @@ -2,8 +2,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" apt-get update && apt-get install -y git wget cmake @@ -17,7 +18,7 @@ root_dir="$(git rev-parse --show-toplevel)" env_dir="${root_dir}/env" lib_dir="${env_dir}/lib" -conda deactivate && conda activate ./env +# conda deactivate (not needed with uv) && source ./.venv/bin/activate # this workflow only tests the libs python -c "import mlagents_envs" diff --git a/.github/unittest/linux_libs/scripts_unity_mlagents/setup_env.sh b/.github/unittest/linux_libs/scripts_unity_mlagents/setup_env.sh index 19c821637de..90129e2f240 100755 --- a/.github/unittest/linux_libs/scripts_unity_mlagents/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_unity_mlagents/setup_env.sh @@ -11,7 +11,7 @@ set -v apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -33,37 +33,32 @@ apt-get upgrade -y libstdc++6 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" # 3. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" -pip install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) diff --git a/.github/unittest/linux_libs/scripts_vd4rl/environment.yml b/.github/unittest/linux_libs/scripts_vd4rl/environment.yml deleted file mode 100644 index 386ec24b3c2..00000000000 --- a/.github/unittest/linux_libs/scripts_vd4rl/environment.yml +++ /dev/null @@ -1,24 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - huggingface_hub - - tqdm - - h5py diff --git a/.github/unittest/linux_libs/scripts_vd4rl/install.sh b/.github/unittest/linux_libs/scripts_vd4rl/install.sh old mode 100755 new mode 100644 index c31b364be09..b2d10a69239 --- a/.github/unittest/linux_libs/scripts_vd4rl/install.sh +++ b/.github/unittest/linux_libs/scripts_vd4rl/install.sh @@ -7,8 +7,15 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +# Add uv to PATH (it was installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -29,15 +36,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu -U + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu -U else - pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -46,16 +53,17 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import functorch;import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation # smoke test python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_vd4rl/post_process.sh b/.github/unittest/linux_libs/scripts_vd4rl/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_vd4rl/post_process.sh +++ b/.github/unittest/linux_libs/scripts_vd4rl/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_vd4rl/requirements.txt b/.github/unittest/linux_libs/scripts_vd4rl/requirements.txt new file mode 100644 index 00000000000..27aee99abf5 --- /dev/null +++ b/.github/unittest/linux_libs/scripts_vd4rl/requirements.txt @@ -0,0 +1,19 @@ +pip +hypothesis +future +cloudpickle +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +huggingface_hub +tqdm +h5py diff --git a/.github/unittest/linux_libs/scripts_vd4rl/run_test.sh b/.github/unittest/linux_libs/scripts_vd4rl/run_test.sh index 83424d83058..81ab4908f19 100755 --- a/.github/unittest/linux_libs/scripts_vd4rl/run_test.sh +++ b/.github/unittest/linux_libs/scripts_vd4rl/run_test.sh @@ -2,8 +2,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" apt-get update && apt-get remove swig -y && apt-get install -y git gcc patchelf libosmesa6-dev libgl1-mesa-glx libglfw3 swig3.0 cmake ln -s /usr/bin/swig3.0 /usr/bin/swig @@ -18,7 +19,7 @@ root_dir="$(git rev-parse --show-toplevel)" env_dir="${root_dir}/env" lib_dir="${env_dir}/lib" -conda deactivate && conda activate ./env +# conda deactivate (not needed with uv) && source ./.venv/bin/activate python .github/unittest/helpers/coverage_run_parallel.py -m pytest test/test_libs.py --instafail -v --durations 200 --capture no -k TestVD4RL --error-for-skips --runslow coverage combine diff --git a/.github/unittest/linux_libs/scripts_vd4rl/setup_env.sh b/.github/unittest/linux_libs/scripts_vd4rl/setup_env.sh index a7afd646646..90129e2f240 100755 --- a/.github/unittest/linux_libs/scripts_vd4rl/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_vd4rl/setup_env.sh @@ -11,7 +11,7 @@ set -v apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -33,37 +33,32 @@ apt-get upgrade -y libstdc++6 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" # 3. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" -pip3 install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) diff --git a/.github/unittest/linux_libs/scripts_vmas/environment.yml b/.github/unittest/linux_libs/scripts_vmas/environment.yml deleted file mode 100644 index 58c4a00400c..00000000000 --- a/.github/unittest/linux_libs/scripts_vmas/environment.yml +++ /dev/null @@ -1,26 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - cloudpickle - - gym - - gym-notices - - numpy - - pyglet==1.5.27 - - six - - torch - - vmas - - zipp - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-error-for-skips - - pytest-asyncio - - expecttest - - pybind11[global] - - pyyaml - - scipy diff --git a/.github/unittest/linux_libs/scripts_vmas/install.sh b/.github/unittest/linux_libs/scripts_vmas/install.sh old mode 100755 new mode 100644 index 137a6cb827a..b7aa004e6d6 --- a/.github/unittest/linux_libs/scripts_vmas/install.sh +++ b/.github/unittest/linux_libs/scripts_vmas/install.sh @@ -7,8 +7,15 @@ unset PYTORCH_VERSION set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +# Add uv to PATH (it was installed in setup_env.sh) +export PATH="$HOME/.local/bin:$PATH" +source "${root_dir}/.venv/bin/activate" + +# Install build dependencies EARLY (required for --no-build-isolation) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" + if [ "${CU_VERSION:-}" == cpu ] ; then version="cpu" @@ -28,15 +35,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with cu128" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch --index-url https://download.pytorch.org/whl/cpu + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu else - pip3 install torch --index-url https://download.pytorch.org/whl/cu128 + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 fi else printf "Failed to install pytorch" @@ -45,14 +52,15 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi # smoke test python -c "import tensordict" + printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation python -c "import torchrl" diff --git a/.github/unittest/linux_libs/scripts_vmas/post_process.sh b/.github/unittest/linux_libs/scripts_vmas/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_libs/scripts_vmas/post_process.sh +++ b/.github/unittest/linux_libs/scripts_vmas/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_libs/scripts_vmas/requirements.txt b/.github/unittest/linux_libs/scripts_vmas/requirements.txt new file mode 100644 index 00000000000..0fdb004c733 --- /dev/null +++ b/.github/unittest/linux_libs/scripts_vmas/requirements.txt @@ -0,0 +1,21 @@ +pip +cloudpickle +gym +gym-notices +numpy +pyglet==1.5.27 +six +torch +vmas +zipp +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-error-for-skips +pytest-asyncio +expecttest +pybind11[global] +pyyaml +scipy diff --git a/.github/unittest/linux_libs/scripts_vmas/run_test.sh b/.github/unittest/linux_libs/scripts_vmas/run_test.sh index fd933bcdf58..89ee1563522 100755 --- a/.github/unittest/linux_libs/scripts_vmas/run_test.sh +++ b/.github/unittest/linux_libs/scripts_vmas/run_test.sh @@ -2,8 +2,9 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +export PATH="$HOME/.local/bin:$PATH" +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" apt-get update && apt-get install -y git wget cmake diff --git a/.github/unittest/linux_libs/scripts_vmas/setup_env.sh b/.github/unittest/linux_libs/scripts_vmas/setup_env.sh index 8cfc8d349ec..dcc84553173 100755 --- a/.github/unittest/linux_libs/scripts_vmas/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_vmas/setup_env.sh @@ -9,7 +9,7 @@ set -e apt-get update && apt-get upgrade -y && apt-get install -y git cmake # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -apt-get install -y wget \ +apt-get install -y wget curl \ gcc \ g++ \ unzip \ @@ -31,37 +31,32 @@ apt-get upgrade -y libstdc++6 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" +# 1. Install uv +printf "* Installing uv +" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" + + +# 2. Create test environment at ./.venv +printf "python: ${PYTHON_VERSION} +" +printf "* Creating a test environment with uv +" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 2. Create test environment at ./env -printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" # 4. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" cat "${this_dir}/environment.yml" -pip install pip --upgrade +uv pip install pip --upgrade -conda env update --file "${this_dir}/environment.yml" --prune +# Dependencies installed via uv pip (see converted script) diff --git a/.github/unittest/linux_olddeps/scripts_gym_0_13/environment.yml b/.github/unittest/linux_olddeps/scripts_gym_0_13/environment.yml deleted file mode 100644 index 84cce31cfe6..00000000000 --- a/.github/unittest/linux_olddeps/scripts_gym_0_13/environment.yml +++ /dev/null @@ -1,30 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - protobuf - - pip: - - hypothesis - - future - - cloudpickle - - gym[atari]==0.13 - - pygame - - moviepy<2.0.0 - - tqdm - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - expecttest - - pyyaml - - scipy - - hydra-core - - mujoco<3.3.6 - - patchelf - - pyopengl==3.1.4 - - ray - - av - - h5py - - numpy<2.0.0 diff --git a/.github/unittest/linux_olddeps/scripts_gym_0_13/install.sh b/.github/unittest/linux_olddeps/scripts_gym_0_13/install.sh index bf74d774d68..92df0bad58e 100755 --- a/.github/unittest/linux_olddeps/scripts_gym_0_13/install.sh +++ b/.github/unittest/linux_olddeps/scripts_gym_0_13/install.sh @@ -48,16 +48,15 @@ fi # install tensordict if [[ "$RELEASE" == 0 ]]; then - conda install anaconda::cmake -y - python3 -m pip install "pybind11[global]" - python3 -m pip install git+https://github.com/pytorch/tensordict.git + python -m pip install "pybind11[global]" + python -m pip install git+https://github.com/pytorch/tensordict.git else - python3 -m pip install tensordict + python -m pip install tensordict fi # smoke test -python3 -c "import tensordict" +python -c "import tensordict" printf "* Installing torchrl\n" -python3 setup.py develop -python3 -c "import torchrl" +python -m pip install -e . --no-build-isolation --no-deps +python -c "import torchrl" diff --git a/.github/unittest/linux_olddeps/scripts_gym_0_13/requirements.txt b/.github/unittest/linux_olddeps/scripts_gym_0_13/requirements.txt new file mode 100644 index 00000000000..219fdacd37a --- /dev/null +++ b/.github/unittest/linux_olddeps/scripts_gym_0_13/requirements.txt @@ -0,0 +1,25 @@ +pip +protobuf +hypothesis +future +cloudpickle +gym[atari]==0.13 +pygame +moviepy<2.0.0 +tqdm +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +expecttest +pyyaml +scipy +hydra-core +mujoco<3.3.6 +patchelf +pyopengl==3.1.4 +ray +av +h5py +numpy<2.0.0 diff --git a/.github/unittest/linux_olddeps/scripts_gym_0_13/setup_env.sh b/.github/unittest/linux_olddeps/scripts_gym_0_13/setup_env.sh index 4c83bdeeaa7..fbb61f01291 100755 --- a/.github/unittest/linux_olddeps/scripts_gym_0_13/setup_env.sh +++ b/.github/unittest/linux_olddeps/scripts_gym_0_13/setup_env.sh @@ -11,8 +11,8 @@ set -v this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" apt-get update && apt-get upgrade -y -printf "* Installing vim - git - wget\n" -apt-get install -y vim git wget +printf "* Installing vim - git - wget - cmake\n" +apt-get install -y vim git wget cmake printf "* Installing glfw - glew - osmesa part 1\n" apt-get install -y libglvnd0 libgl1 libglx0 libegl1 libgles2 xvfb libx11-dev libegl-dev diff --git a/.github/unittest/linux_optdeps/scripts/environment.yml b/.github/unittest/linux_optdeps/scripts/environment.yml deleted file mode 100644 index 3c16131fec9..00000000000 --- a/.github/unittest/linux_optdeps/scripts/environment.yml +++ /dev/null @@ -1,21 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - pytest-timeout - - expecttest - - pybind11[global] - - pyyaml - - scipy - - coverage - - ray diff --git a/.github/unittest/linux_optdeps/scripts/post_process.sh b/.github/unittest/linux_optdeps/scripts/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_optdeps/scripts/post_process.sh +++ b/.github/unittest/linux_optdeps/scripts/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_optdeps/scripts/requirements.txt b/.github/unittest/linux_optdeps/scripts/requirements.txt new file mode 100644 index 00000000000..4a5ed37a27b --- /dev/null +++ b/.github/unittest/linux_optdeps/scripts/requirements.txt @@ -0,0 +1,16 @@ +pip +hypothesis +future +cloudpickle +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +pytest-timeout +expecttest +pybind11[global] +pyyaml +scipy +coverage +ray diff --git a/.github/unittest/linux_optdeps/scripts/run_all.sh b/.github/unittest/linux_optdeps/scripts/run_all.sh index fd309018be0..f6ff3a445b0 100755 --- a/.github/unittest/linux_optdeps/scripts/run_all.sh +++ b/.github/unittest/linux_optdeps/scripts/run_all.sh @@ -10,7 +10,7 @@ set -e if [[ $OSTYPE != 'darwin'* ]]; then apt-get update && apt-get upgrade -y - apt-get install -y vim git wget cmake + apt-get install -y vim git wget curl cmake apt-get install -y libglfw3 libgl1-mesa-glx libosmesa6 libglew-dev apt-get install -y libglvnd0 libgl1 libglx0 libegl1 libgles2 @@ -39,41 +39,36 @@ fi # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" -lib_dir="${env_dir}/lib" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac +# 1. Install uv +printf "* Installing uv\n" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" - -# 2. Create test environment at ./env +# 2. Create test environment at ./.venv printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" +printf "* Creating a test environment with uv\n" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" -# 3. Install Conda dependencies -printf "* Installing dependencies (except PyTorch)\n" -echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" -cat "${this_dir}/environment.yml" +# 3. Install dependencies for OPTDEPS testing +# Optdeps tests that torchrl works WITHOUT optional dependencies +# We install: build deps + test deps only (NO optional deps like gym, transformers, etc.) +printf "* Installing build and test dependencies (NO optional deps)\n" -pip3 install pip --upgrade +# Install build dependencies FIRST (required for C++ extensions) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" -conda env update --file "${this_dir}/environment.yml" --prune +# Install test dependencies from requirements.txt (pytest, hypothesis, etc.) +printf "* Installing test dependencies from requirements.txt\n" +uv pip install -r "${this_dir}/requirements.txt" + +# Install pip for compatibility with packages that expect it +uv pip install pip # ============================================================================================ # # ================================ PyTorch & TorchRL ========================================= # @@ -99,15 +94,15 @@ git submodule sync && git submodule update --init --recursive printf "Installing PyTorch with %s\n" "${CU_VERSION}" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/$CU_VERSION -U + uv pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/$CU_VERSION -U fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch --index-url https://download.pytorch.org/whl/cpu -U + uv pip install torch --index-url https://download.pytorch.org/whl/cpu -U else - pip3 install torch --index-url https://download.pytorch.org/whl/$CU_VERSION -U + uv pip install torch --index-url https://download.pytorch.org/whl/$CU_VERSION -U fi else printf "Failed to install pytorch" @@ -117,22 +112,18 @@ fi # smoke test python -c "import functorch" -## install snapshot -#if [[ "$TORCH_VERSION" == "nightly" ]]; then -# pip3 install git+https://github.com/pytorch/torchsnapshot -#else -# pip3 install torchsnapshot -#fi - # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi -printf "* Installing torchrl\n" -python setup.py develop +printf "* Installing torchrl (will install CORE dependencies only)\n" +# Standard installation: installs [project.dependencies] but NOT [project.optional-dependencies] +# Core deps: torch, numpy, packaging, cloudpickle, tensordict (all pre-installed above) +# --no-build-isolation: build deps already available, avoids reinstalling torch in temp env +uv pip install -e . --no-build-isolation # smoke test python -c "import torchrl" @@ -141,8 +132,8 @@ python -c "import torchrl" # ================================ Run tests ========================================= # -# find libstdc -STDC_LOC=$(find conda/ -name "libstdc++.so.6" | head -1) +# find libstdc (if needed) +STDC_LOC=$(find ${env_dir}/ -name "libstdc++.so.6" 2>/dev/null | head -1 || echo "") export PYTORCH_TEST_WITH_SLOW='1' export LAZY_LEGACY_OP=False diff --git a/.github/unittest/linux_sota/scripts/environment.yml b/.github/unittest/linux_sota/scripts/environment.yml deleted file mode 100644 index 848720a7bbb..00000000000 --- a/.github/unittest/linux_sota/scripts/environment.yml +++ /dev/null @@ -1,33 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - protobuf - - pip: - - hypothesis - - future - - cloudpickle - - pygame - - moviepy<2.0.0 - - tqdm - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - expecttest - - pybind11[global] - - pyyaml - - scipy - - hydra-core - - imageio==2.26.0 - - dm_control - - mujoco<3.3.6 - - mlflow - - av - - coverage - - vmas - - transformers - - minari - - minari[create] diff --git a/.github/unittest/linux_sota/scripts/post_process.sh b/.github/unittest/linux_sota/scripts/post_process.sh index e97bf2a7b1b..6647ab38833 100755 --- a/.github/unittest/linux_sota/scripts/post_process.sh +++ b/.github/unittest/linux_sota/scripts/post_process.sh @@ -2,5 +2,5 @@ set -e -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env +root_dir="$(git rev-parse --show-toplevel)" +source "${root_dir}/.venv/bin/activate" diff --git a/.github/unittest/linux_sota/scripts/requirements.txt b/.github/unittest/linux_sota/scripts/requirements.txt new file mode 100644 index 00000000000..5ca1f09bb42 --- /dev/null +++ b/.github/unittest/linux_sota/scripts/requirements.txt @@ -0,0 +1,28 @@ +pip +protobuf +hypothesis +future +cloudpickle +pygame +moviepy<2.0.0 +tqdm +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +expecttest +pybind11[global] +pyyaml +scipy +hydra-core +imageio==2.26.0 +dm_control +mujoco<3.3.6 +mlflow +av +coverage +vmas +transformers +minari +minari[create] diff --git a/.github/unittest/linux_sota/scripts/run_all.sh b/.github/unittest/linux_sota/scripts/run_all.sh index d7681e433f7..720a8198e03 100755 --- a/.github/unittest/linux_sota/scripts/run_all.sh +++ b/.github/unittest/linux_sota/scripts/run_all.sh @@ -8,7 +8,7 @@ set -v apt-get update && apt-get upgrade -y -apt-get install -y vim git wget cmake +apt-get install -y vim git wget curl cmake apt-get install -y libglfw3 libgl1-mesa-glx libosmesa6 libglew-dev libosmesa6-dev apt-get install -y libglvnd0 libgl1 libglx0 libegl1 libgles2 @@ -25,93 +25,64 @@ cp $this_dir/10_nvidia.json /usr/share/glvnd/egl_vendor.d/10_nvidia.json # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" -lib_dir="${env_dir}/lib" +env_dir="${root_dir}/.venv" cd "${root_dir}" -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac +# 1. Install uv +printf "* Installing uv\n" +curl -LsSf https://astral.sh/uv/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" - -# 2. Create test environment at ./env +# 2. Create test environment at ./.venv printf "python: ${PYTHON_VERSION}\n" -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" +printf "* Creating a test environment with uv\n" +uv venv "${env_dir}" --python="${PYTHON_VERSION}" +source "${env_dir}/bin/activate" # 3. Install mujoco printf "* Installing mujoco and related\n" mkdir -p $root_dir/.mujoco cd $root_dir/.mujoco/ -#wget https://github.com/deepmind/mujoco/releases/download/2.1.1/mujoco-2.1.1-linux-x86_64.tar.gz -#tar -xf mujoco-2.1.1-linux-x86_64.tar.gz wget https://mujoco.org/download/mujoco210-linux-x86_64.tar.gz tar -xf mujoco210-linux-x86_64.tar.gz cd "${root_dir}" -# 4. Install Conda dependencies +# 4. Install dependencies (except PyTorch) printf "* Installing dependencies (except PyTorch)\n" -echo " - python=${PYTHON_VERSION}" >> "${this_dir}/environment.yml" -cat "${this_dir}/environment.yml" export MUJOCO_PY_MUJOCO_PATH=$root_dir/.mujoco/mujoco210 -#export MJLIB_PATH=$root_dir/.mujoco/mujoco-2.1.1/lib/libmujoco.so.2.1.1 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$root_dir/.mujoco/mujoco210/bin export SDL_VIDEODRIVER=dummy export MUJOCO_GL=egl export PYOPENGL_PLATFORM=egl export LAZY_LEGACY_OP=False export COMPOSITE_LP_AGGREGATE=0 +export MAX_IDLE_COUNT=1000 +export DISPLAY=:99 +export BATCHED_PIPE_TIMEOUT=60 +export TOKENIZERS_PARALLELISM=true -conda env config vars set \ - MAX_IDLE_COUNT=1000 \ - MUJOCO_PY_MUJOCO_PATH=$root_dir/.mujoco/mujoco210 \ - DISPLAY=:99 \ - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$root_dir/.mujoco/mujoco210/bin \ - SDL_VIDEODRIVER=dummy \ - MUJOCO_GL=egl \ - PYOPENGL_PLATFORM=egl \ - BATCHED_PIPE_TIMEOUT=60 \ - TOKENIZERS_PARALLELISM=true - -pip install pip --upgrade +# Install build dependencies FIRST (required for C++ extensions) +printf "* Installing build dependencies\n" +uv pip install setuptools wheel ninja "pybind11[global]" -conda env update --file "${this_dir}/environment.yml" --prune +# Install test dependencies and libraries from requirements.txt +printf "* Installing dependencies from requirements.txt\n" +uv pip install -r "${this_dir}/requirements.txt" -conda deactivate -conda activate "${env_dir}" +# Install pip for compatibility with packages that expect it +uv pip install pip # install d4rl -pip install free-mujoco-py -pip install git+https://github.com/Farama-Foundation/d4rl@master#egg=d4rl - -# TODO: move this down -- will break torchrl installation -conda install -y -c conda-forge libstdcxx-ng=12 -## find libstdc -STDC_LOC=$(find conda/ -name "libstdc++.so.6" | head -1) -conda env config vars set \ - MAX_IDLE_COUNT=1000 \ - LD_PRELOAD=${root_dir}/$STDC_LOC TOKENIZERS_PARALLELISM=true +# Install poetry first (free-mujoco-py needs it as build backend) +uv pip install poetry +uv pip install free-mujoco-py +uv pip install git+https://github.com/Farama-Foundation/d4rl@master#egg=d4rl # compile mujoco-py (bc it's done at runtime for whatever reason someone thought it was a good idea) python -c """import gym;import d4rl""" -# install ale-py: manylinux names are broken for CentOS so we need to manually download and -# rename them - # ============================================================================================ # # ================================ PyTorch & TorchRL ========================================= # @@ -127,21 +98,21 @@ version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" # submodules git submodule sync && git submodule update --init --recursive -pip3 install ale-py -U -pip3 install "gym[atari,accept-rom-license]" "gymnasium>=1.1.0" -U +uv pip install ale-py -U +uv pip install "gym[atari,accept-rom-license]" "gymnasium>=1.1.0" -U printf "Installing PyTorch with %s\n" "${CU_VERSION}" if [[ "$TORCH_VERSION" == "nightly" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install --pre torch torchvision numpy==1.26.4 --index-url https://download.pytorch.org/whl/nightly/cpu -U + uv pip install --pre torch torchvision numpy==1.26.4 --index-url https://download.pytorch.org/whl/nightly/cpu -U else - pip3 install --pre torch torchvision numpy==1.26.4 --index-url https://download.pytorch.org/whl/nightly/$CU_VERSION + uv pip install --pre torch torchvision numpy==1.26.4 --index-url https://download.pytorch.org/whl/nightly/$CU_VERSION fi elif [[ "$TORCH_VERSION" == "stable" ]]; then if [ "${CU_VERSION:-}" == cpu ] ; then - pip3 install torch torchvision numpy==1.26.4 --index-url https://download.pytorch.org/whl/cpu + uv pip install torch torchvision numpy==1.26.4 --index-url https://download.pytorch.org/whl/cpu else - pip3 install torch torchvision numpy==1.26.4 --index-url https://download.pytorch.org/whl/$CU_VERSION + uv pip install torch torchvision numpy==1.26.4 --index-url https://download.pytorch.org/whl/$CU_VERSION fi else printf "Failed to install pytorch" @@ -151,18 +122,15 @@ fi # smoke test python -c "import functorch" -## install snapshot -#pip install git+https://github.com/pytorch/torchsnapshot - # install tensordict if [[ "$RELEASE" == 0 ]]; then - pip3 install git+https://github.com/pytorch/tensordict.git + uv pip install git+https://github.com/pytorch/tensordict.git else - pip3 install tensordict + uv pip install tensordict fi printf "* Installing torchrl\n" -python setup.py develop +uv pip install -e . --no-build-isolation # ==================================================================================== # # ================================ Run tests ========================================= # diff --git a/.github/unittest/windows_optdepts/scripts/environment.yml b/.github/unittest/windows_optdepts/scripts/environment.yml deleted file mode 100644 index 2740c77f434..00000000000 --- a/.github/unittest/windows_optdepts/scripts/environment.yml +++ /dev/null @@ -1,18 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pip - - pip: - - hypothesis - - future - - cloudpickle - - pytest - - pytest-cov - - pytest-mock - - pytest-instafail - - pytest-rerunfailures - - expecttest - - pyyaml - - scipy - - coverage diff --git a/.github/unittest/windows_optdepts/scripts/requirements.txt b/.github/unittest/windows_optdepts/scripts/requirements.txt new file mode 100644 index 00000000000..1ca4637f52e --- /dev/null +++ b/.github/unittest/windows_optdepts/scripts/requirements.txt @@ -0,0 +1,13 @@ +pip +hypothesis +future +cloudpickle +pytest +pytest-cov +pytest-mock +pytest-instafail +pytest-rerunfailures +expecttest +pyyaml +scipy +coverage diff --git a/pyproject.toml b/pyproject.toml index 9159f05f9a0..e8ee0890fa3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools", "wheel", "torch", "ninja", "numpy", "pybind11[global]", "cmake"] +requires = ["setuptools", "wheel", "torch", "ninja", "numpy", "pybind11[global]"] build-backend = "setuptools.build_meta" [project] @@ -53,6 +53,7 @@ tests = [ "pytest-benchmark", "pytest-rerunfailures", "pytest-error-for-skips", + "virtualenv", # Required by Ray's runtime_env when using pip installations ] utils = [ "tensorboard", diff --git a/test/test_env.py b/test/test_env.py index fe0c52dce22..c6b9e241797 100644 --- a/test/test_env.py +++ b/test/test_env.py @@ -13,6 +13,7 @@ import pickle import random import re +import time from collections import defaultdict from functools import partial from sys import platform @@ -3693,13 +3694,23 @@ def test_batched_nondynamic(self, penv): use_buffers=True, mp_start_method=mp_ctx if penv is ParallelEnv else None, ) - env_buffers.set_seed(0) - torch.manual_seed(0) - rollout_buffers = env_buffers.rollout( - 20, return_contiguous=True, break_when_any_done=False - ) - del env_buffers + try: + env_buffers.set_seed(0) + torch.manual_seed(0) + rollout_buffers = env_buffers.rollout( + 20, return_contiguous=True, break_when_any_done=False + ) + finally: + env_buffers.close(raise_if_closed=False) + del env_buffers gc.collect() + # Add a small delay to allow multiprocessing resource_sharer threads + # to fully clean up before creating the next environment. This prevents + # a race condition where the old resource_sharer service thread is still + # active when the new environment starts, causing a deadlock. + # See: https://bugs.python.org/issue30289 + if penv is ParallelEnv: + time.sleep(0.1) env_no_buffers = penv( 3, @@ -3707,12 +3718,15 @@ def test_batched_nondynamic(self, penv): use_buffers=False, mp_start_method=mp_ctx if penv is ParallelEnv else None, ) - env_no_buffers.set_seed(0) - torch.manual_seed(0) - rollout_no_buffers = env_no_buffers.rollout( - 20, return_contiguous=True, break_when_any_done=False - ) - del env_no_buffers + try: + env_no_buffers.set_seed(0) + torch.manual_seed(0) + rollout_no_buffers = env_no_buffers.rollout( + 20, return_contiguous=True, break_when_any_done=False + ) + finally: + env_no_buffers.close(raise_if_closed=False) + del env_no_buffers gc.collect() assert_allclose_td(rollout_buffers, rollout_no_buffers)