fix: fix some minor permission issue. Add detailed explanation to the running example run_multinode_train.sh

Author: jianhan-amd

multi_node/run_multinode_train.sh

@@ -1,20 +1,23 @@
 #!/bin/bash
 
 # Required environment variables for wan_multinode_train.sh
+
+# core42-4-a08u25:172.29.0.73
 export COORDINATOR_IP=172.29.0.73
-export IMAGE_TAG=jianhan-wan-multinode-train:v1
-export MULTI_NODES_LOG_DIR=/home/amd/jianhan/multi_node_log
-export SHARE_DOCKERFILE_PATH=/home/amd/jianhan/github/maxdiffusion/multi_node/docker/jax_maxdiffusion_wan2.1_train_inference.ubuntu.amd.Dockerfile
-export SHARED_CODE_BASE_PATH=/home/amd/jianhan/github/maxdiffusion
+export IMAGE_TAG=your-name-wan-multinode-train:v1
+# Please keep MULTI_NODES_LOG_DIR outside SHARED_CODE_BASE_PATH since we are going to sync the whole SHARED_CODE_BASE_PATH
+export MULTI_NODES_LOG_DIR=/home/amd/your_dir/multi_node_log
+export SHARE_DOCKERFILE_PATH=/home/amd/your_dir/maxdiffusion/multi_node/docker/jax_maxdiffusion_wan2.1_train_inference.ubuntu.amd.Dockerfile
+export SHARED_CODE_BASE_PATH=/home/amd/your_dir/maxdiffusion
 export MAXDIFFUSION_DIR_IN_DOCKER=/app/maxdiffusion
-export RUN_NAME=WAN_1_3B_FSDP8
+export RUN_NAME=WAN_14B_FSDP8
 export REMOVE_IMAGES=n
 export REGISTRY_USERNAME=""
 export REGISTRY_TOKEN=""
 export CHMOD_RUN=n
 
 # Define node list
-NODES="core42-5-a08u01,core42-1-a08u07,core42-3-a08u19,core42-4-a08u25"
+NODES="core42-1-a08u07,core42-3-a08u19,core42-4-a08u25,core42-5-a08u01"
 
 # 1. Clean and sync codebase
 # To remove Docker images during cleanup, uncomment the line below:
@@ -24,4 +27,7 @@ bash wan_multinode_train.sh "$NODES" clean
 bash wan_multinode_train.sh "$NODES" build
 
 # # 3. Launch training
+# Please put the JAX COORDINATOR to the first of the list. The JAX COORDINATOR node will be launched before others to make sure all nodes can connect to the JAX COORDINATOR service.
+# core42-4-a08u25:172.29.0.73
+NODES="core42-4-a08u25,core42-1-a08u07,core42-3-a08u19,core42-5-a08u01"
 bash wan_multinode_train.sh "$NODES" launch

multi_node/wan_multinode_train.sbatch

@@ -104,7 +104,7 @@ echo "Building the container image on all nodes"
 srun bash -c '
     MAX_RETRIES=5
     INITIAL_DELAY=30 # seconds
-    MAX_DELAY=1800 # seconds
+    MAX_DELAY=180 # seconds
     RETRY_COUNT=0
 
     while true; do

multi_node/wan_multinode_train_clean.sh

@@ -161,7 +161,7 @@ for node in "${NODES[@]}"; do
         continue
     }
 
-    if rsync -az --delete --info=progress2 -e "ssh" \
+    if rsync -az --delete --omit-dir-times --info=progress2 -e "ssh" \
         "$SHARED_CODE_BASE_PATH/" "$node:$SHARED_CODE_BASE_PATH/" \
         > "${LOG_DIR}/sync_${node}.log" 2>&1; then
         echo "[${node}] ✓ Sync completed"

multi_node/wan_multinode_train_launch.sh

@@ -52,7 +52,7 @@ fi
 readonly IMAGE_TAG="${IMAGE_TAG}"
 readonly COORDINATOR_IP="${COORDINATOR_IP}"
 readonly JAX_COORDINATOR_PORT=12345
-readonly COORDINATOR_TIMEOUT=1800
+readonly COORDINATOR_TIMEOUT=180
 
 # Paths configuration (should be set by wrapper script)
 readonly MULTI_NODES_LOG_DIR="${MULTI_NODES_LOG_DIR}"