fix(nightly): fix flow-grpo and sft-gpt-oss nightlies

research/flow_grpo/tests/test_suites/common.env

@@ -45,13 +45,39 @@ run_flow_grpo_recipe() {
   cd "$PROJECT_ROOT"
   NEMO_RL_ROOT=$(realpath "$PROJECT_ROOT/../..")
 
-  # paddleocr pulls the full opencv wheels, whose cv2 needs these system libs.
-  if ! ldconfig -p | grep -q libxcb.so.1; then
-    apt-get update && apt-get install -y --no-install-recommends libxcb1 libgl1 libglib2.0-0
-  fi
-
   uv sync --locked --inexact --group test
 
+  # paddleocr pulls the full opencv wheels, whose cv2 needs these system libs on every node.
+  uv run --no-sync python - <<'EOF'
+import subprocess
+
+import ray
+from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
+
+ray.init(address="auto")
+
+
+@ray.remote(num_cpus=0)
+def install_cv2_libs():
+    subprocess.run(
+        "ldconfig -p | grep -q libxcb.so.1 || "
+        "(apt-get update && apt-get install -y --no-install-recommends libxcb1 libgl1 libglib2.0-0)",
+        shell=True,
+        check=True,
+    )
+
+
+ray.get(
+    [
+        install_cv2_libs.options(
+            scheduling_strategy=NodeAffinitySchedulingStrategy(n["NodeID"], soft=False)
+        ).remote()
+        for n in ray.nodes()
+        if n["Alive"]
+    ]
+)
+EOF
+
   # The recipe inherits data.{train,val}.prompt_file from the exemplar config;
   # generate the OCR prompt files on first use.
   if [[ ! -f data/ocr/train_prompts.jsonl ]]; then

tests/test_suites/llm/sft-gpt-oss-20b-1n8g-automodel-ep8.sh

@@ -35,9 +35,9 @@ uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS
 if [[ $(jq 'to_entries | .[] | select(.key == "train/loss") | .value | keys | map(tonumber) | max' $JSON_METRICS) -ge $MAX_STEPS ]]; then
     uv run tests/check_metrics.py $JSON_METRICS \
         'data["train/loss"]["1"] < 7.0' \
-        'data["train/loss"]["50"] < 0.4' \
-        'data["train/grad_norm"]["50"] < 17.5' \
-        'data["train/grad_norm"]["50"] > 10.0'
+        'data["train/loss"]["50"] < 0.35' \
+        'data["train/grad_norm"]["50"] < 14.0' \
+        'data["train/grad_norm"]["50"] > 8.0'
 
     # Clean up checkpoint directory after successful run to save space.
     rm -rf "$CKPT_DIR"