Skip to content

fix: normalize agent IDs and remove bootstrap files for benchmark #37

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
olearycrew merged 4 commits into from
Mar 19, 2026
Merged

fix: normalize agent IDs and remove bootstrap files for benchmark #37

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0cccb85
fix: normalize agent IDs and remove bootstrap files for benchmark
Mar 9, 2026
46445b2
fix: copy skills to benchmark workspace and add delay before grading
Mar 9, 2026
11b2dca
Merge branch 'main' into fix/agent-id-normalization
zhuanghaoz Mar 13, 2026
0306e85
Apply suggestions from code review
olearycrew Mar 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 36 additions & 7 deletions scripts/lib_agent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ def _get_agent_workspace(agent_id: str) -> Path | None:
return None

# Parse the agent list output to find workspace
# OpenClaw normalizes colons to dashes in agent names, so check both.
normalized_id = agent_id.replace(":", "-")
# OpenClaw normalizes colons to dashes and lowercases agent names
normalized_id = agent_id.replace(":", "-").lower()
lines = list_result.stdout.split("\n")
found_agent = False
for line in lines:
Expand Down Expand Up @@ -93,9 +93,9 @@ def ensure_agent_exists(agent_id: str, model_id: str, workspace_dir: Path) -> bo
# Extract agent name: "- bench-foo-4-5" or "- main (default)"
name_part = line[2:].split()[0] if line[2:].strip() else ""
if name_part:
existing_agents.add(name_part)
normalized_id = agent_id.replace(":", "-")
if agent_id in existing_agents or normalized_id in existing_agents:
existing_agents.add(name_part.lower())
normalized_id = agent_id.replace(":", "-").lower()
if agent_id.lower() in existing_agents or normalized_id in existing_agents:
# Agent exists — check if workspace matches
current_workspace = _get_agent_workspace(agent_id)
if (
Expand Down Expand Up @@ -208,15 +208,44 @@ def prepare_task_workspace(skill_dir: Path, run_id: str, task: Task, agent_id: s
logger.error("Workspace file not found: %s", source)
raise

# Remove bootstrap files that would trigger the onboarding flow
# These interfere with benchmark tasks
for bootstrap_file in ["BOOTSTRAP.md", "SOUL.md", "USER.md", "IDENTITY.md"]:
bootstrap_path = workspace / bootstrap_file
if bootstrap_path.exists():
try:
bootstrap_path.unlink()
logger.info("Removed bootstrap file: %s", bootstrap_file)
except OSError as exc:
logger.warning("Failed to remove %s: %s", bootstrap_file, exc)

# Copy skills from main workspace to benchmark workspace
# This enables benchmark agents to use installed skills like nano-pdf
main_skills_dir = Path.home() / ".openclaw" / "workspace" / "skills"
if main_skills_dir.exists():
dest_skills_dir = workspace / "skills"
dest_skills_dir.mkdir(parents=True, exist_ok=True)
for skill_dir_src in main_skills_dir.iterdir():
if skill_dir_src.is_dir():
dest_skill_dir = dest_skills_dir / skill_dir_src.name
# Copy skill directory
import shutil
Copy link
Contributor

@kilo-code-bot kilo-code-bot bot Mar 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

WARNING: Redundant import shutil inside a loop — shutil is already imported at line 180 at the top of this same function. While Python caches module imports so this won't cause a runtime error, it's unnecessary and clutters the loop body. Move it to the top of the function alongside the existing import, or simply remove this line.

if dest_skill_dir.exists():
shutil.rmtree(dest_skill_dir)
shutil.copytree(skill_dir_src, dest_skill_dir)
Copy link
Contributor

@kilo-code-bot kilo-code-bot bot Mar 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

WARNING: shutil.copytree follows symlinks by default. If any skill directory under ~/.openclaw/workspace/skills/ contains symlinks pointing to sensitive locations (e.g., ~/.ssh, credentials), those files will be fully copied into the benchmark workspace and become accessible to the benchmark agent.

Consider using shutil.copytree(skill_dir_src, dest_skill_dir, symlinks=True) to preserve symlinks without following them, or ignore_dangling_symlinks=True at minimum. This hardens the copy against unintended data exposure in the benchmark workspace.

logger.info("Copied skill to benchmark workspace: %s", skill_dir_src.name)

return workspace


def _get_agent_store_dir(agent_id: str) -> Path:
base_dir = Path.home() / ".openclaw" / "agents"
# OpenClaw normalizes agent IDs to lowercase and replaces colons with dashes
normalized_id = agent_id.replace(":", "-").lower()
direct_dir = base_dir / agent_id
if direct_dir.exists():
return direct_dir
normalized_dir = base_dir / agent_id.replace(":", "-")
normalized_dir = base_dir / normalized_id
if normalized_dir.exists():
return normalized_dir
return direct_dir
Expand All @@ -235,7 +264,7 @@ def _resolve_session_id_from_store(agent_id: str) -> str | None:
if not isinstance(sessions_payload, dict):
return None

normalized_id = agent_id.replace(":", "-")
normalized_id = agent_id.replace(":", "-").lower()
preferred_keys = [
f"agent:{agent_id}:main",
f"agent:{agent_id}:default",
Expand Down