Fixed install_deps so that it happens before each test run + checking for the deps that are required separately

.github/workflows/main_python.yml

@@ -30,7 +30,7 @@ jobs:
         # pip install --pre torch torchvision -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html
     - name: Run Tests
       run: |
-        ./run_python_examples.sh "install_deps,run_all,clean"
+        ./run_python_examples.sh "run_all,clean"
     - name: Open issue on failure
       if: ${{ failure() && github.event_name  == 'schedule' }}
       uses: rishabhgupta/git-action-issue@v2

CONTRIBUTING.md

@@ -38,12 +38,11 @@ If you're new, we encourage you to take a look at issues tagged with [good first
 ## For bug fixes
 
 1. Fork the repo and create your branch from `main`.
-2. Make sure you have a GPU-enabled machine, either locally or in the cloud. `g4dn.4xlarge` is a good starting point on AWS.
-3. Make your code change.
-4. First, install all dependencies with `./run_python_examples.sh "install_deps"`.
-5. Then, make sure that `./run_python_examples.sh` passes locally by running the script end to end.
-6. If you haven't already, complete the Contributor License Agreement ("CLA").
-7. Address any feedback in code review promptly.
+1. Make sure you have a GPU-enabled machine, either locally or in the cloud. `g4dn.4xlarge` is a good starting point on AWS.
+1. Make your code change.
+1. Then, make sure that `./run_python_examples.sh` passes locally by running the script end to end.
+1. If you haven't already, complete the Contributor License Agreement ("CLA").
+1. Address any feedback in code review promptly.
 
 ## Contributor License Agreement ("CLA")
 

run_distributed_examples.sh

@@ -6,8 +6,8 @@
 #
 # Optionally specify a comma separated list of examples to run.
 # can be run as:
-# ./run_python_examples.sh "install_deps,run_all,clean"
-# to pip install dependencies (other than pytorch), run all examples, and remove temporary/changed data files.
+# ./run_python_examples.sh "run_all,clean"
+# run all examples, and remove temporary/changed data files.
 # Expects pytorch, torchvision to be installed.
 
 BASE_DIR="$(pwd)/$(dirname $0)"

run_python_examples.sh

@@ -1,18 +1,42 @@
 #!/usr/bin/env bash
+set -eo pipefail
+DEBUG=${DEBUG:-false}
+[[ $DEBUG == true ]] && set -x
 #
 # This script runs through the code in each of the python examples.
 # The purpose is just as an integration test, not to actually train models in any meaningful way.
 # For that reason, most of these set epochs = 1 and --dry-run.
 #
 # Optionally specify a comma separated list of examples to run.
 # can be run as:
-# ./run_python_examples.sh "install_deps,run_all,clean"
-# to pip install dependencies (other than pytorch), run all examples, and remove temporary/changed data files.
+# ./run_python_examples.sh "run_all,clean"
+# run all examples, and remove temporary/changed data files.
 # Expects pytorch, torchvision to be installed.
 
 BASE_DIR="$(pwd)/$(dirname $0)"
 source $BASE_DIR/utils.sh
 
+echo "] Running Python examples"
+# Check if required packages are installed
+echo "Checking for required packages..."
+if ! pip show torch; then
+  echo "torch is not installed. Please install PyTorch."
+  exit 1
+fi
+
+if ! pip show torchvision; then
+  echo "torchvision is not installed. Please install torchvision."
+  exit 1
+fi
+
+if ! pip show pillow; then
+  echo "Pillow is not installed. Please install Pillow."
+  exit 1
+fi
+
+echo "All required packages are installed!"
+
+echo "Checking CUDA availability"
 USE_CUDA=$(python -c "import torchvision, torch; print(torch.cuda.is_available())")
 case $USE_CUDA in
   "True")
@@ -215,9 +239,11 @@ if [ "" == "$EXAMPLES" ]; then
 else
   for i in $(echo $EXAMPLES | sed "s/,/ /g")
   do
+    echo "==============="
     echo "Starting $i"
     $i
     echo "Finished $i, status $?"
+    echo "==============="
   done
 fi
 

utils.sh

@@ -22,17 +22,20 @@ function error() {
 }
 
 function install_deps() {
-  echo "installing requirements"
-  cat $BASE_DIR/*/requirements.txt | \
-    sort -u | \
+  EXAMPLE_NAME=$1
+  echo "] $EXAMPLE_NAME: installing requirements"
+  [[ -f requirements.txt ]] || return
+  for req in $(cat requirements.txt); do
     # testing the installed version of torch, so don't pip install it.
-    grep -vE '^torch$' | \
-    pip install -r /dev/stdin || \
-    { error "failed to install dependencies"; exit 1; }
+    if [[ "$req" != "torch" ]]; then
+      pip install "$req" || { error "failed to install $req"; exit 1; }
+    fi
+  done
 }
 
 function start() {
-  EXAMPLE=${FUNCNAME[1]}
-  cd $BASE_DIR/$EXAMPLE
-  echo "Running example: $EXAMPLE"
+  EXAMPLE_NAME=${FUNCNAME[1]}
+  cd $BASE_DIR/$EXAMPLE_NAME
+  install_deps $EXAMPLE_NAME
+  echo "] $EXAMPLE_NAME: running"
 }