diff --git a/.github/workflows/main_python.yml b/.github/workflows/main_python.yml index ffc0c4deff..1f97ad5a05 100644 --- a/.github/workflows/main_python.yml +++ b/.github/workflows/main_python.yml @@ -30,7 +30,7 @@ jobs: # pip install --pre torch torchvision -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html - name: Run Tests run: | - ./run_python_examples.sh "install_deps,run_all,clean" + ./run_python_examples.sh "run_all,clean" - name: Open issue on failure if: ${{ failure() && github.event_name == 'schedule' }} uses: rishabhgupta/git-action-issue@v2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e0e0e5ac90..a1afa66dc6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,12 +38,11 @@ If you're new, we encourage you to take a look at issues tagged with [good first ## For bug fixes 1. Fork the repo and create your branch from `main`. -2. Make sure you have a GPU-enabled machine, either locally or in the cloud. `g4dn.4xlarge` is a good starting point on AWS. -3. Make your code change. -4. First, install all dependencies with `./run_python_examples.sh "install_deps"`. -5. Then, make sure that `./run_python_examples.sh` passes locally by running the script end to end. -6. If you haven't already, complete the Contributor License Agreement ("CLA"). -7. Address any feedback in code review promptly. +1. Make sure you have a GPU-enabled machine, either locally or in the cloud. `g4dn.4xlarge` is a good starting point on AWS. +1. Make your code change. +1. Then, make sure that `./run_python_examples.sh` passes locally by running the script end to end. +1. If you haven't already, complete the Contributor License Agreement ("CLA"). +1. Address any feedback in code review promptly. ## Contributor License Agreement ("CLA") diff --git a/run_distributed_examples.sh b/run_distributed_examples.sh index c2260d7c78..9069a57cfb 100755 --- a/run_distributed_examples.sh +++ b/run_distributed_examples.sh @@ -6,8 +6,8 @@ # # Optionally specify a comma separated list of examples to run. # can be run as: -# ./run_python_examples.sh "install_deps,run_all,clean" -# to pip install dependencies (other than pytorch), run all examples, and remove temporary/changed data files. +# ./run_python_examples.sh "run_all,clean" +# run all examples, and remove temporary/changed data files. # Expects pytorch, torchvision to be installed. BASE_DIR="$(pwd)/$(dirname $0)" diff --git a/run_python_examples.sh b/run_python_examples.sh index 0e06e4cfc0..3ad8d5f032 100755 --- a/run_python_examples.sh +++ b/run_python_examples.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash +set -eo pipefail +DEBUG=${DEBUG:-false} +[[ $DEBUG == true ]] && set -x # # This script runs through the code in each of the python examples. # The purpose is just as an integration test, not to actually train models in any meaningful way. @@ -6,13 +9,34 @@ # # Optionally specify a comma separated list of examples to run. # can be run as: -# ./run_python_examples.sh "install_deps,run_all,clean" -# to pip install dependencies (other than pytorch), run all examples, and remove temporary/changed data files. +# ./run_python_examples.sh "run_all,clean" +# run all examples, and remove temporary/changed data files. # Expects pytorch, torchvision to be installed. BASE_DIR="$(pwd)/$(dirname $0)" source $BASE_DIR/utils.sh +echo "] Running Python examples" +# Check if required packages are installed +echo "Checking for required packages..." +if ! pip show torch; then + echo "torch is not installed. Please install PyTorch." + exit 1 +fi + +if ! pip show torchvision; then + echo "torchvision is not installed. Please install torchvision." + exit 1 +fi + +if ! pip show pillow; then + echo "Pillow is not installed. Please install Pillow." + exit 1 +fi + +echo "All required packages are installed!" + +echo "Checking CUDA availability" USE_CUDA=$(python -c "import torchvision, torch; print(torch.cuda.is_available())") case $USE_CUDA in "True") @@ -215,9 +239,11 @@ if [ "" == "$EXAMPLES" ]; then else for i in $(echo $EXAMPLES | sed "s/,/ /g") do + echo "===============" echo "Starting $i" $i echo "Finished $i, status $?" + echo "===============" done fi diff --git a/utils.sh b/utils.sh index b7ed613e6e..8ca86140c1 100644 --- a/utils.sh +++ b/utils.sh @@ -22,17 +22,20 @@ function error() { } function install_deps() { - echo "installing requirements" - cat $BASE_DIR/*/requirements.txt | \ - sort -u | \ + EXAMPLE_NAME=$1 + echo "] $EXAMPLE_NAME: installing requirements" + [[ -f requirements.txt ]] || return + for req in $(cat requirements.txt); do # testing the installed version of torch, so don't pip install it. - grep -vE '^torch$' | \ - pip install -r /dev/stdin || \ - { error "failed to install dependencies"; exit 1; } + if [[ "$req" != "torch" ]]; then + pip install "$req" || { error "failed to install $req"; exit 1; } + fi + done } function start() { - EXAMPLE=${FUNCNAME[1]} - cd $BASE_DIR/$EXAMPLE - echo "Running example: $EXAMPLE" + EXAMPLE_NAME=${FUNCNAME[1]} + cd $BASE_DIR/$EXAMPLE_NAME + install_deps $EXAMPLE_NAME + echo "] $EXAMPLE_NAME: running" }