Skip to content

Commit 7a5459f

Browse files
committed
different approach
1 parent c1d80cb commit 7a5459f

1 file changed

Lines changed: 63 additions & 47 deletions

File tree

‎benchmarks/benchmark_lib.sh‎

Lines changed: 63 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -444,59 +444,75 @@ with open('${CONFIG_FILE}', 'w') as f:
444444
print('[profile] Done')
445445
" || echo "[profile] Warning: profiling injection failed"
446446

447-
# Copy profiling patches into srt-slurm scripts dir (mounted at /srtctl-benchmarks in container)
448-
local patches_src="${GITHUB_WORKSPACE:-$(dirname "$0")/..}/patches"
449-
local srt_scripts_dir="src/srtctl/benchmarks/scripts"
450-
if [[ -d "$patches_src" && -d "$srt_scripts_dir" ]]; then
451-
cp -v "$patches_src"/*-profiling.py "$srt_scripts_dir/"
452-
echo "[profile] Copied profiling patches to $srt_scripts_dir/"
453-
fi
454-
455-
# For TRT-LLM: install sitecustomize.py that auto-applies the profiling patch
456-
# when Python starts inside the container. The scripts dir is mounted at
457-
# /srtctl-benchmarks, and we add it to PYTHONPATH via the config YAML.
458-
if [[ "${FRAMEWORK:-}" == *"trt"* && -d "$srt_scripts_dir" ]]; then
459-
mkdir -p "$srt_scripts_dir/profiling_sitecustomize"
460-
cat > "$srt_scripts_dir/profiling_sitecustomize/sitecustomize.py" <<'SITEPY'
461-
import os, sys
462-
_mode = os.environ.get("PROFILING_MODE", "")
463-
print(f"[sitecustomize] loaded, PROFILING_MODE={_mode}, PYTHONPATH={os.environ.get('PYTHONPATH','')}", file=sys.stderr, flush=True)
464-
if _mode:
465-
patch_path = "/srtctl-benchmarks/trtllm-profiling.py"
466-
print(f"[sitecustomize] patch exists: {os.path.isfile(patch_path)}", file=sys.stderr, flush=True)
467-
if os.path.isfile(patch_path):
468-
try:
469-
import importlib.util
470-
spec = importlib.util.spec_from_file_location("_trtllm_profiling", patch_path)
471-
mod = importlib.util.module_from_spec(spec)
472-
spec.loader.exec_module(mod)
473-
if hasattr(mod, "patch"):
474-
mod.patch()
475-
print("[sitecustomize] patch applied successfully", file=sys.stderr, flush=True)
476-
else:
477-
print("[sitecustomize] patch module has no patch() function", file=sys.stderr, flush=True)
478-
except Exception as e:
479-
print(f"[sitecustomize] patch failed: {e}", file=sys.stderr, flush=True)
480-
else:
481-
print(f"[sitecustomize] patch file not found at {patch_path}", file=sys.stderr, flush=True)
482-
SITEPY
483-
# Inject PYTHONPATH into the srtctl config so the container picks up sitecustomize
447+
# For TRT-LLM: use a setup script to directly patch handler_base.py inside the container.
448+
# The setup_script runs inside the container before the worker starts.
449+
if [[ "${FRAMEWORK:-}" == *"trt"* ]]; then
450+
local patches_src="${GITHUB_WORKSPACE:-$(dirname "$0")/..}/patches"
451+
local configs_dir="configs"
452+
mkdir -p "$configs_dir"
453+
454+
# Copy the patch to configs dir (mounted at /configs in container)
455+
if [[ -f "$patches_src/trtllm-profiling.py" ]]; then
456+
cp "$patches_src/trtllm-profiling.py" "$configs_dir/"
457+
echo "[profile] Copied trtllm-profiling.py to $configs_dir/"
458+
fi
459+
460+
# Create a setup script that patches handler_base.py directly
461+
cat > "$configs_dir/apply-profiling-patch.sh" <<'SETUP'
462+
#!/bin/bash
463+
echo "[profile-setup] Applying TRT-LLM profiling patch..."
464+
465+
PATCH_SRC="/configs/trtllm-profiling.py"
466+
HANDLER="/opt/dynamo/venv/lib/python3.12/site-packages/dynamo/trtllm/request_handlers/handler_base.py"
467+
468+
if [[ ! -f "$PATCH_SRC" ]]; then
469+
echo "[profile-setup] ERROR: patch not found at $PATCH_SRC"
470+
exit 0
471+
fi
472+
473+
if [[ ! -f "$HANDLER" ]]; then
474+
# Try to find it
475+
HANDLER=$(find /opt -name "handler_base.py" -path "*/trtllm/*" 2>/dev/null | head -1)
476+
if [[ -z "$HANDLER" ]]; then
477+
echo "[profile-setup] ERROR: handler_base.py not found"
478+
exit 0
479+
fi
480+
fi
481+
482+
echo "[profile-setup] Found handler at: $HANDLER"
483+
484+
# Append the patch import and call to the end of handler_base.py
485+
# The patch installs a post-import hook that wraps generate() with torch.profiler
486+
cat >> "$HANDLER" <<'PATCH_APPEND'
487+
488+
# --- InferenceX profiling patch (auto-appended) ---
489+
import os as _os
490+
if _os.environ.get("PROFILING_MODE"):
491+
import importlib.util as _ilu
492+
_patch_path = "/configs/trtllm-profiling.py"
493+
if _os.path.isfile(_patch_path):
494+
_spec = _ilu.spec_from_file_location("_trtllm_profiling", _patch_path)
495+
_mod = _ilu.module_from_spec(_spec)
496+
_spec.loader.exec_module(_mod)
497+
if hasattr(_mod, "patch"):
498+
_mod.patch()
499+
# --- End profiling patch ---
500+
PATCH_APPEND
501+
502+
echo "[profile-setup] Patch appended to $HANDLER"
503+
SETUP
504+
chmod +x "$configs_dir/apply-profiling-patch.sh"
505+
506+
# Inject setup_script into srtctl config
484507
python3 -c "
485508
import yaml
486509
with open('${CONFIG_FILE}') as f:
487510
cfg = yaml.safe_load(f)
488-
# Add PYTHONPATH to backend environment for both prefill and decode
489-
backend = cfg.get('backend', {})
490-
for env_key in ['prefill_environment', 'decode_environment']:
491-
env = backend.get(env_key, {})
492-
existing = env.get('PYTHONPATH', '')
493-
env['PYTHONPATH'] = '/srtctl-benchmarks/profiling_sitecustomize' + (':' + existing if existing else '')
494-
backend[env_key] = env
495-
cfg['backend'] = backend
511+
cfg['setup_script'] = 'apply-profiling-patch.sh'
496512
with open('${CONFIG_FILE}', 'w') as f:
497513
yaml.dump(cfg, f, default_flow_style=False)
498-
print('[profile] Injected PYTHONPATH for TRT-LLM profiling patch')
499-
" || echo "[profile] Warning: failed to inject PYTHONPATH"
514+
print('[profile] Set setup_script=apply-profiling-patch.sh')
515+
" || echo "[profile] Warning: failed to inject setup_script"
500516
fi
501517

502518
# For SGLang: enable layerwise NVTX markers via sglang_config CLI args

0 commit comments

Comments
 (0)