@@ -444,59 +444,75 @@ with open('${CONFIG_FILE}', 'w') as f:
444444print('[profile] Done')
445445" || echo " [profile] Warning: profiling injection failed"
446446
447- # Copy profiling patches into srt-slurm scripts dir (mounted at /srtctl-benchmarks in container)
448- local patches_src=" ${GITHUB_WORKSPACE:- $(dirname " $0 " )/ ..} /patches"
449- local srt_scripts_dir=" src/srtctl/benchmarks/scripts"
450- if [[ -d " $patches_src " && -d " $srt_scripts_dir " ]]; then
451- cp -v " $patches_src " /* -profiling.py " $srt_scripts_dir /"
452- echo " [profile] Copied profiling patches to $srt_scripts_dir /"
453- fi
454-
455- # For TRT-LLM: install sitecustomize.py that auto-applies the profiling patch
456- # when Python starts inside the container. The scripts dir is mounted at
457- # /srtctl-benchmarks, and we add it to PYTHONPATH via the config YAML.
458- if [[ " ${FRAMEWORK:- } " == * " trt" * && -d " $srt_scripts_dir " ]]; then
459- mkdir -p " $srt_scripts_dir /profiling_sitecustomize"
460- cat > " $srt_scripts_dir /profiling_sitecustomize/sitecustomize.py" << 'SITEPY '
461- import os, sys
462- _mode = os.environ.get("PROFILING_MODE", "")
463- print(f"[sitecustomize] loaded, PROFILING_MODE={_mode}, PYTHONPATH={os.environ.get('PYTHONPATH','')}", file=sys.stderr, flush=True)
464- if _mode:
465- patch_path = "/srtctl-benchmarks/trtllm-profiling.py"
466- print(f"[sitecustomize] patch exists: {os.path.isfile(patch_path)}", file=sys.stderr, flush=True)
467- if os.path.isfile(patch_path):
468- try:
469- import importlib.util
470- spec = importlib.util.spec_from_file_location("_trtllm_profiling", patch_path)
471- mod = importlib.util.module_from_spec(spec)
472- spec.loader.exec_module(mod)
473- if hasattr(mod, "patch"):
474- mod.patch()
475- print("[sitecustomize] patch applied successfully", file=sys.stderr, flush=True)
476- else:
477- print("[sitecustomize] patch module has no patch() function", file=sys.stderr, flush=True)
478- except Exception as e:
479- print(f"[sitecustomize] patch failed: {e}", file=sys.stderr, flush=True)
480- else:
481- print(f"[sitecustomize] patch file not found at {patch_path}", file=sys.stderr, flush=True)
482- SITEPY
483- # Inject PYTHONPATH into the srtctl config so the container picks up sitecustomize
447+ # For TRT-LLM: use a setup script to directly patch handler_base.py inside the container.
448+ # The setup_script runs inside the container before the worker starts.
449+ if [[ " ${FRAMEWORK:- } " == * " trt" * ]]; then
450+ local patches_src=" ${GITHUB_WORKSPACE:- $(dirname " $0 " )/ ..} /patches"
451+ local configs_dir=" configs"
452+ mkdir -p " $configs_dir "
453+
454+ # Copy the patch to configs dir (mounted at /configs in container)
455+ if [[ -f " $patches_src /trtllm-profiling.py" ]]; then
456+ cp " $patches_src /trtllm-profiling.py" " $configs_dir /"
457+ echo " [profile] Copied trtllm-profiling.py to $configs_dir /"
458+ fi
459+
460+ # Create a setup script that patches handler_base.py directly
461+ cat > " $configs_dir /apply-profiling-patch.sh" << 'SETUP '
462+ #!/bin/bash
463+ echo "[profile-setup] Applying TRT-LLM profiling patch..."
464+
465+ PATCH_SRC="/configs/trtllm-profiling.py"
466+ HANDLER="/opt/dynamo/venv/lib/python3.12/site-packages/dynamo/trtllm/request_handlers/handler_base.py"
467+
468+ if [[ ! -f "$PATCH_SRC" ]]; then
469+ echo "[profile-setup] ERROR: patch not found at $PATCH_SRC"
470+ exit 0
471+ fi
472+
473+ if [[ ! -f "$HANDLER" ]]; then
474+ # Try to find it
475+ HANDLER=$(find /opt -name "handler_base.py" -path "*/trtllm/*" 2>/dev/null | head -1)
476+ if [[ -z "$HANDLER" ]]; then
477+ echo "[profile-setup] ERROR: handler_base.py not found"
478+ exit 0
479+ fi
480+ fi
481+
482+ echo "[profile-setup] Found handler at: $HANDLER"
483+
484+ # Append the patch import and call to the end of handler_base.py
485+ # The patch installs a post-import hook that wraps generate() with torch.profiler
486+ cat >> "$HANDLER" <<'PATCH_APPEND'
487+
488+ # --- InferenceX profiling patch (auto-appended) ---
489+ import os as _os
490+ if _os.environ.get("PROFILING_MODE"):
491+ import importlib.util as _ilu
492+ _patch_path = "/configs/trtllm-profiling.py"
493+ if _os.path.isfile(_patch_path):
494+ _spec = _ilu.spec_from_file_location("_trtllm_profiling", _patch_path)
495+ _mod = _ilu.module_from_spec(_spec)
496+ _spec.loader.exec_module(_mod)
497+ if hasattr(_mod, "patch"):
498+ _mod.patch()
499+ # --- End profiling patch ---
500+ PATCH_APPEND
501+
502+ echo "[profile-setup] Patch appended to $HANDLER"
503+ SETUP
504+ chmod +x " $configs_dir /apply-profiling-patch.sh"
505+
506+ # Inject setup_script into srtctl config
484507 python3 -c "
485508import yaml
486509with open('${CONFIG_FILE} ') as f:
487510 cfg = yaml.safe_load(f)
488- # Add PYTHONPATH to backend environment for both prefill and decode
489- backend = cfg.get('backend', {})
490- for env_key in ['prefill_environment', 'decode_environment']:
491- env = backend.get(env_key, {})
492- existing = env.get('PYTHONPATH', '')
493- env['PYTHONPATH'] = '/srtctl-benchmarks/profiling_sitecustomize' + (':' + existing if existing else '')
494- backend[env_key] = env
495- cfg['backend'] = backend
511+ cfg['setup_script'] = 'apply-profiling-patch.sh'
496512with open('${CONFIG_FILE} ', 'w') as f:
497513 yaml.dump(cfg, f, default_flow_style=False)
498- print('[profile] Injected PYTHONPATH for TRT-LLM profiling patch')
499- " || echo " [profile] Warning: failed to inject PYTHONPATH "
514+ print('[profile] Set setup_script=apply- profiling- patch.sh ')
515+ " || echo " [profile] Warning: failed to inject setup_script "
500516 fi
501517
502518 # For SGLang: enable layerwise NVTX markers via sglang_config CLI args
0 commit comments